<commit_before>package handler

import (
	"database/sql"
	"fmt"
	"net/http"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/jmoiron/sqlx"
	"github.com/lib/pq"
	common "github.com/ooni/orchestra/common"
)

// DomainFrontedCollector is a {"domain": "a", "front": "b"} map
type DomainFrontedCollector struct {
	Domain string `json:"domain"`
	Front  string `json:"front"`
}

// CollectorInfo holds the type and address of a collector
type CollectorInfo struct {
	Type    string `json:"type"`
	Address string `json:"address"`
}

// GetCollectors returns the list of collectors available
func GetCollectors(types string, db *sqlx.DB) ([]CollectorInfo, error) {
	var (
		err  error
		args []interface{}
	)
	collectors := make([]CollectorInfo, 0)

	query := fmt.Sprintf(`SELECT
		type,
		address,
		front_domain
		FROM %s`,
		pq.QuoteIdentifier(common.CollectorsTable))
	if types != "" {
		query += " WHERE type = ANY($1)"
		args = append(args, pq.StringArray(strings.Split(types, ",")))
	}
	rows, err := db.Query(query)
	if err != nil {
		if err == sql.ErrNoRows {
			return collectors, nil
		}
		ctx.WithError(err).Error("failed to get collectors")
		return collectors, err
	}
	defer rows.Close()
	for rows.Next() {
		var (
			ctype    string
			caddress string
			cfront   sql.NullString
		)
		err = rows.Scan(&ctype, &caddress, &cfront)
		if err != nil {
			ctx.WithError(err).Error("failed to get collector row")
			// In this case we fail fast
			return collectors, err
		}
		if ctype == "domain_fronted" {
			if !cfront.Valid {
				ctx.Error("domain_fronted collector with bad front domain")
				return collectors, err
			}
			caddress = fmt.Sprintf("%s@%s", caddress, cfront.String)
		}
		collectors = append(collectors, CollectorInfo{
			Type:    ctype,
			Address: caddress,
		})
	}
	return collectors, nil
}

// TestHelperInfo holds the name, type and address of a test helper
type TestHelperInfo struct {
	Name    string `json:"name"`
	Type    string `json:"type"`
	Address string `json:"address"`
}

// GetTestHelpers returns a list of test helpers
func GetTestHelpers(names string, db *sqlx.DB) ([]TestHelperInfo, error) {
	var (
		err  error
		args []interface{}
	)
	testHelpers := make([]TestHelperInfo, 0)
	query := fmt.Sprintf(`SELECT
		name,
		type,
		address
		FROM %s`,
		pq.QuoteIdentifier(common.TestHelpersTable))
	if names != "" {
		query += " WHERE name = ANY($1)"
		args = append(args, pq.StringArray(strings.Split(names, ",")))
	}
	rows, err := db.Query(query, args...)
	if err != nil {
		if err == sql.ErrNoRows {
			return testHelpers, nil
		}
		ctx.WithError(err).Error("failed to get test helpers")
		return testHelpers, err
	}
	defer rows.Close()
	for rows.Next() {
		var testHelper TestHelperInfo
		err = rows.Scan(&testHelper.Name, &testHelper.Address, &testHelper.Type)
		if err != nil {
			ctx.WithError(err).Error("failed to get test_helper row")
			continue
		}
		testHelpers = append(testHelpers, testHelper)
	}
	return testHelpers, nil
}

// CollectorsHandler returns the list of requested collectors
func CollectorsHandler(c *gin.Context) {
	db := c.MustGet("DB").(*sqlx.DB)

	types := c.Query("types")
	collectors, err := GetCollectors(types, db)
	if err != nil {
		c.JSON(http.StatusInternalServerError,
			gin.H{"error": "server side error"})
		return
	}

	c.JSON(http.StatusOK,
		gin.H{"results": collectors})
	return
}

// TestHelpersHandler returns the list of requested test helpers
func TestHelpersHandler(c *gin.Context) {
	db := c.MustGet("DB").(*sqlx.DB)

	names := c.Query("names")
	testHelpers, err := GetTestHelpers(names, db)
	if err != nil {
		c.JSON(http.StatusInternalServerError,
			gin.H{"error": "server side error"})
		return
	}

	c.JSON(http.StatusOK,
		gin.H{"results": testHelpers})
	return
}
<commit_msg>Fix https://github.com/ooni/orchestra/issues/54<commit_after>package handler

import (
	"database/sql"
	"fmt"
	"net/http"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/jmoiron/sqlx"
	"github.com/lib/pq"
	common "github.com/ooni/orchestra/common"
)

// DomainFrontedCollector is a {"domain": "a", "front": "b"} map
type DomainFrontedCollector struct {
	Domain string `json:"domain"`
	Front  string `json:"front"`
}

// CollectorInfo holds the type and address of a collector
type CollectorInfo struct {
	Type    string `json:"type"`
	Address string `json:"address"`
}

// GetCollectors returns the list of collectors available
func GetCollectors(types string, db *sqlx.DB) ([]CollectorInfo, error) {
	var (
		err  error
		args []interface{}
	)
	collectors := make([]CollectorInfo, 0)

	query := fmt.Sprintf(`SELECT
		type,
		address,
		front_domain
		FROM %s`,
		pq.QuoteIdentifier(common.CollectorsTable))
	if types != "" {
		query += " WHERE type = ANY($1)"
		args = append(args, pq.StringArray(strings.Split(types, ",")))
	}
	rows, err := db.Query(query)
	if err != nil {
		if err == sql.ErrNoRows {
			return collectors, nil
		}
		ctx.WithError(err).Error("failed to get collectors")
		return collectors, err
	}
	defer rows.Close()
	for rows.Next() {
		var (
			ctype    string
			caddress string
			cfront   sql.NullString
		)
		err = rows.Scan(&ctype, &caddress, &cfront)
		if err != nil {
			ctx.WithError(err).Error("failed to get collector row")
			// In this case we fail fast
			return collectors, err
		}
		if ctype == "domain_fronted" {
			if !cfront.Valid {
				ctx.Error("domain_fronted collector with bad front domain")
				return collectors, err
			}
			caddress = fmt.Sprintf("%s@%s", caddress, cfront.String)
		}
		collectors = append(collectors, CollectorInfo{
			Type:    ctype,
			Address: caddress,
		})
	}
	return collectors, nil
}

// TestHelperInfo holds the name, type and address of a test helper
type TestHelperInfo struct {
	Name    string `json:"name"`
	Type    string `json:"type"`
	Address string `json:"address"`
}

// GetTestHelpers returns a list of test helpers
func GetTestHelpers(names string, db *sqlx.DB) ([]TestHelperInfo, error) {
	var (
		err  error
		args []interface{}
	)
	testHelpers := make([]TestHelperInfo, 0)
	query := fmt.Sprintf(`SELECT
		name,
		type,
		address
		FROM %s`,
		pq.QuoteIdentifier(common.TestHelpersTable))
	if names != "" {
		query += " WHERE name = ANY($1)"
		args = append(args, pq.StringArray(strings.Split(names, ",")))
	}
	rows, err := db.Query(query, args...)
	if err != nil {
		if err == sql.ErrNoRows {
			return testHelpers, nil
		}
		ctx.WithError(err).Error("failed to get test helpers")
		return testHelpers, err
	}
	defer rows.Close()
	for rows.Next() {
		var testHelper TestHelperInfo
		err = rows.Scan(&testHelper.Name, &testHelper.Type, &testHelper.Address)
		if err != nil {
			ctx.WithError(err).Error("failed to get test_helper row")
			continue
		}
		testHelpers = append(testHelpers, testHelper)
	}
	return testHelpers, nil
}

// CollectorsHandler returns the list of requested collectors
func CollectorsHandler(c *gin.Context) {
	db := c.MustGet("DB").(*sqlx.DB)

	types := c.Query("types")
	collectors, err := GetCollectors(types, db)
	if err != nil {
		c.JSON(http.StatusInternalServerError,
			gin.H{"error": "server side error"})
		return
	}

	c.JSON(http.StatusOK,
		gin.H{"results": collectors})
	return
}

// TestHelpersHandler returns the list of requested test helpers
func TestHelpersHandler(c *gin.Context) {
	db := c.MustGet("DB").(*sqlx.DB)

	names := c.Query("names")
	testHelpers, err := GetTestHelpers(names, db)
	if err != nil {
		c.JSON(http.StatusInternalServerError,
			gin.H{"error": "server side error"})
		return
	}

	c.JSON(http.StatusOK,
		gin.H{"results": testHelpers})
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nofilesystem

package collector

import (
	"bytes"

	"github.com/go-kit/kit/log/level"
	"golang.org/x/sys/unix"
)

const (
	defIgnoredMountPoints = "^/(dev)($|/)"
	defIgnoredFSTypes     = "^devfs$"
	readOnly              = 0x1 // MNT_RDONLY
	noWait                = 0x2 // MNT_NOWAIT
)

// Expose filesystem fullness.
func (c *filesystemCollector) GetStats() ([]filesystemStats, error) {
	n, err := unix.Getfsstat(nil, noWait)
	if err != nil {
		return nil, err
	}
	buf := make([]unix.Statfs_t, n)
	_, err = unix.Getfsstat(buf, noWait)
	if err != nil {
		return nil, err
	}
	stats := []filesystemStats{}
	for _, fs := range buf {
		// We need to work out the lengths of the actual strings here,
		// otherwuse we will end up with null bytes in our label values.
		mountpoint_len := bytes.Index(fs.Mntonname[:], []byte{0})
		mountpoint := string(fs.Mntonname[:mountpoint_len])
		if c.ignoredMountPointsPattern.MatchString(mountpoint) {
			level.Debug(c.logger).Log("msg", "Ignoring mount point", "mountpoint", mountpoint)
			continue
		}

		device_len := bytes.Index(fs.Mntfromname[:], []byte{0})
		fstype_len := bytes.Index(fs.Fstypename[:], []byte{0})
		device := string(fs.Mntfromname[:device_len])
		fstype := string(fs.Fstypename[:fstype_len])
		if c.ignoredFSTypesPattern.MatchString(fstype) {
			level.Debug(c.logger).Log("msg", "Ignoring fs type", "type", fstype)
			continue
		}

		var ro float64
		if (fs.Flags & readOnly) != 0 {
			ro = 1
		}

		stats = append(stats, filesystemStats{
			labels: filesystemLabels{
				device:     device,
				mountPoint: rootfsStripPrefix(mountpoint),
				fsType:     fstype,
			},
			size:      float64(fs.Blocks) * float64(fs.Bsize),
			free:      float64(fs.Bfree) * float64(fs.Bsize),
			avail:     float64(fs.Bavail) * float64(fs.Bsize),
			files:     float64(fs.Files),
			filesFree: float64(fs.Ffree),
			ro:        ro,
		})
	}
	return stats, nil
}
<commit_msg>filesystem_freebsd: Use bytesToString to get label values<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nofilesystem

package collector

import (
	"github.com/go-kit/kit/log/level"
	"golang.org/x/sys/unix"
)

const (
	defIgnoredMountPoints = "^/(dev)($|/)"
	defIgnoredFSTypes     = "^devfs$"
	readOnly              = 0x1 // MNT_RDONLY
	noWait                = 0x2 // MNT_NOWAIT
)

// Expose filesystem fullness.
func (c *filesystemCollector) GetStats() ([]filesystemStats, error) {
	n, err := unix.Getfsstat(nil, noWait)
	if err != nil {
		return nil, err
	}
	buf := make([]unix.Statfs_t, n)
	_, err = unix.Getfsstat(buf, noWait)
	if err != nil {
		return nil, err
	}
	stats := []filesystemStats{}
	for _, fs := range buf {
		mountpoint := bytesToString(fs.Mntonname[:])
		if c.ignoredMountPointsPattern.MatchString(mountpoint) {
			level.Debug(c.logger).Log("msg", "Ignoring mount point", "mountpoint", mountpoint)
			continue
		}

		device := bytesToString(fs.Mntfromname[:])
		fstype := bytesToString(fs.Fstypename[:])
		if c.ignoredFSTypesPattern.MatchString(fstype) {
			level.Debug(c.logger).Log("msg", "Ignoring fs type", "type", fstype)
			continue
		}

		var ro float64
		if (fs.Flags & readOnly) != 0 {
			ro = 1
		}

		stats = append(stats, filesystemStats{
			labels: filesystemLabels{
				device:     device,
				mountPoint: rootfsStripPrefix(mountpoint),
				fsType:     fstype,
			},
			size:      float64(fs.Blocks) * float64(fs.Bsize),
			free:      float64(fs.Bfree) * float64(fs.Bsize),
			avail:     float64(fs.Bavail) * float64(fs.Bsize),
			files:     float64(fs.Files),
			filesFree: float64(fs.Ffree),
			ro:        ro,
		})
	}
	return stats, nil
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package pkg

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	log "github.com/Sirupsen/logrus"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/newt/newtutil"
	"mynewt.apache.org/newt/newt/repo"
	"mynewt.apache.org/newt/util"
	"mynewt.apache.org/newt/viper"
	"mynewt.apache.org/newt/yaml"
)

var PackageHashIgnoreDirs = map[string]bool{
	"obj": true,
	"bin": true,
	".":   true,
}

var LocalPackageSpecialNames = map[string]bool{
	"src":     true,
	"include": true,
	"bin":     true,
}

type LocalPackage struct {
	repo        *repo.Repo
	name        string
	basePath    string
	packageType interfaces.PackageType

	// General information about the package
	desc *PackageDesc
	// Dependencies for this package
	deps []*Dependency

	// Package init function name and stage.  These are used to generate the
	// sysinit C file.
	initFnName string
	initStage  int

	// Extra package-specific settings that don't come from syscfg.  For
	// example, SELFTEST gets set when the newt test command is used.
	injectedSettings map[string]string

	// Pointer to pkg.yml configuration structure
	Viper *viper.Viper

	// Names of all source yml files; used to determine if rebuild required.
	cfgFilenames []string
}

func NewLocalPackage(r *repo.Repo, pkgDir string) *LocalPackage {
	pkg := &LocalPackage{
		desc: &PackageDesc{},
		// XXX: Initialize viper object; clients should not need to check for
		// nil pointer.
		repo:             r,
		basePath:         filepath.Clean(pkgDir) + "/", // XXX: Remove slash.
		injectedSettings: map[string]string{},
	}
	return pkg
}

func (pkg *LocalPackage) Name() string {
	return pkg.name
}

func (pkg *LocalPackage) FullName() string {
	r := pkg.Repo()
	if r.IsLocal() {
		return pkg.Name()
	} else {
		return newtutil.BuildPackageString(r.Name(), pkg.Name())
	}
}

func (pkg *LocalPackage) BasePath() string {
	return filepath.Clean(pkg.basePath)
}

func (pkg *LocalPackage) Type() interfaces.PackageType {
	return pkg.packageType
}

func (pkg *LocalPackage) Repo() interfaces.RepoInterface {
	return pkg.repo
}

func (pkg *LocalPackage) Desc() *PackageDesc {
	return pkg.desc
}

func (pkg *LocalPackage) SetName(name string) {
	pkg.name = name
	// XXX: Also set "pkg.name" in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetBasePath(basePath string) {
	pkg.basePath = basePath
}

func (pkg *LocalPackage) SetType(packageType interfaces.PackageType) {
	pkg.packageType = packageType
	// XXX: Also set "pkg.type" in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetDesc(desc *PackageDesc) {
	pkg.desc = desc
	// XXX: Also set desc fields in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetRepo(r *repo.Repo) {
	pkg.repo = r
}

func (pkg *LocalPackage) Hash() (string, error) {
	hash := sha1.New()

	err := filepath.Walk(pkg.basePath,
		func(path string, info os.FileInfo, err error) error {
			name := info.Name()
			if PackageHashIgnoreDirs[name] {
				return filepath.SkipDir
			}

			if info.IsDir() {
				// SHA the directory name into the hash
				hash.Write([]byte(name))
			} else {
				// SHA the file name & contents into the hash
				contents, err := ioutil.ReadFile(path)
				if err != nil {
					return err
				}
				hash.Write(contents)
			}
			return nil
		})
	if err != nil && err != filepath.SkipDir {
		return "", util.NewNewtError(err.Error())
	}

	hashStr := fmt.Sprintf("%x", hash.Sum(nil))

	return hashStr, nil
}

func (pkg *LocalPackage) CfgFilenames() []string {
	return pkg.cfgFilenames
}

func (pkg *LocalPackage) AddCfgFilename(cfgFilename string) {
	pkg.cfgFilenames = append(pkg.cfgFilenames, cfgFilename)
}

func (pkg *LocalPackage) HasDep(searchDep *Dependency) bool {
	for _, dep := range pkg.deps {
		if dep.String() == searchDep.String() {
			return true
		}
	}
	return false
}

func (pkg *LocalPackage) AddDep(dep *Dependency) {
	// Remember the name of the configuration file so that it can be specified
	// as a dependency to the compiler.
	pkg.deps = append(pkg.deps, dep)
}

func (pkg *LocalPackage) Deps() []*Dependency {
	return pkg.deps
}

func (pkg *LocalPackage) readDesc(v *viper.Viper) (*PackageDesc, error) {
	pdesc := &PackageDesc{}

	pdesc.Author = v.GetString("pkg.author")
	pdesc.Homepage = v.GetString("pkg.homepage")
	pdesc.Description = v.GetString("pkg.description")
	pdesc.Keywords = v.GetStringSlice("pkg.keywords")

	return pdesc, nil
}

func (pkg *LocalPackage) sequenceString(key string) string {
	var buffer bytes.Buffer

	if pkg.Viper != nil {
		for _, f := range pkg.Viper.GetStringSlice(key) {
			buffer.WriteString("    - " + yaml.EscapeString(f) + "\n")
		}
	}

	if buffer.Len() == 0 {
		return ""
	} else {
		return key + ":\n" + buffer.String()
	}
}

// Saves the package's pkg.yml file.
// NOTE: This does not save every field in the package.  Only the fields
// necessary for creating a new target get saved.
func (pkg *LocalPackage) Save() error {
	dirpath := pkg.BasePath()
	if err := os.MkdirAll(dirpath, 0755); err != nil {
		return util.NewNewtError(err.Error())
	}

	filepath := dirpath + "/" + PACKAGE_FILE_NAME
	file, err := os.Create(filepath)
	if err != nil {
		return util.NewNewtError(err.Error())
	}
	defer file.Close()

	file.WriteString("### Package: " + pkg.Name() + "\n")

	// XXX: Just iterate viper object's settings rather than calling out
	// cached settings individually.
	file.WriteString("pkg.name: " + yaml.EscapeString(pkg.Name()) + "\n")
	file.WriteString("pkg.type: " +
		yaml.EscapeString(PackageTypeNames[pkg.Type()]) + "\n")
	file.WriteString("pkg.description: " +
		yaml.EscapeString(pkg.Desc().Description) + "\n")
	file.WriteString("pkg.author: " +
		yaml.EscapeString(pkg.Desc().Author) + "\n")
	file.WriteString("pkg.homepage: " +
		yaml.EscapeString(pkg.Desc().Homepage) + "\n")

	file.WriteString("\n")

	file.WriteString(pkg.sequenceString("pkg.aflags"))
	file.WriteString(pkg.sequenceString("pkg.cflags"))
	file.WriteString(pkg.sequenceString("pkg.features"))
	file.WriteString(pkg.sequenceString("pkg.lflags"))

	return nil
}

// Load reads everything that isn't identity specific into the
// package
func (pkg *LocalPackage) Load() error {
	// Load configuration
	log.Debugf("Loading configuration for package %s", pkg.basePath)

	v, err := util.ReadConfig(pkg.basePath,
		strings.TrimSuffix(PACKAGE_FILE_NAME, ".yml"))
	if err != nil {
		return err
	}
	pkg.Viper = v

	// Set package name from the package
	pkg.name = v.GetString("pkg.name")

	typeString := v.GetString("pkg.type")
	pkg.packageType = PACKAGE_TYPE_LIB
	for t, n := range PackageTypeNames {
		if typeString == n {
			pkg.packageType = t
			break
		}
	}

	pkg.initFnName = v.GetString("pkg.init_function")
	pkg.initStage = v.GetInt("pkg.init_stage")

	// Read the package description from the file
	pkg.desc, err = pkg.readDesc(v)
	if err != nil {
		return err
	}

	pkg.AddCfgFilename(pkg.basePath + PACKAGE_FILE_NAME)

	return nil
}

func (pkg *LocalPackage) InitStage() int {
	return pkg.initStage
}

func (pkg *LocalPackage) InitFnName() string {
	return pkg.initFnName
}

func (pkg *LocalPackage) InjectedSettings() map[string]string {
	return pkg.injectedSettings
}

func (pkg *LocalPackage) Clone(newRepo *repo.Repo,
	newName string) *LocalPackage {

	// XXX: Validate name.

	// Copy the package.
	newPkg := *pkg
	newPkg.repo = newRepo
	newPkg.name = newName
	newPkg.basePath = newRepo.Path() + "/" + newPkg.name

	// Insert the clone into the global package map.
	proj := interfaces.GetProject()
	pMap := proj.PackageList()
	(*pMap[newRepo.Name()])[newPkg.name] = &newPkg

	return &newPkg
}

func LoadLocalPackage(repo *repo.Repo, pkgDir string) (*LocalPackage, error) {
	pkg := NewLocalPackage(repo, pkgDir)
	err := pkg.Load()
	return pkg, err
}

func LocalPackageSpecialName(dirName string) bool {
	_, ok := LocalPackageSpecialNames[dirName]
	return ok
}

func ReadLocalPackageRecursive(repo *repo.Repo,
	pkgList map[string]interfaces.PackageInterface, basePath string,
	pkgName string) error {

	dirList, err := repo.FilteredSearchList(pkgName)
	if err != nil {
		return util.NewNewtError(err.Error())
	}

	for _, name := range dirList {
		if LocalPackageSpecialName(name) || strings.HasPrefix(name, ".") {
			continue
		}

		if err := ReadLocalPackageRecursive(repo, pkgList, basePath,
			filepath.Join(pkgName, name)); err != nil {
			return err
		}
	}

	if util.NodeNotExist(filepath.Join(basePath, pkgName, PACKAGE_FILE_NAME)) {
		return nil
	}

	pkg, err := LoadLocalPackage(repo, filepath.Join(basePath, pkgName))
	if err != nil {
		return err
	}

	if oldPkg, ok := pkgList[pkg.Name()]; ok {
		oldlPkg := oldPkg.(*LocalPackage)
		return util.FmtNewtError("Multiple packages with same pkg.name=%s "+
			"in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(),
			oldlPkg.BasePath(), pkg.BasePath())
	}

	pkgList[pkg.Name()] = pkg

	return nil
}

func ReadLocalPackages(repo *repo.Repo,
	basePath string) (*map[string]interfaces.PackageInterface, error) {

	pkgList := map[string]interfaces.PackageInterface{}

	searchPaths, err := repo.FilteredSearchList("")
	if err != nil {
		return nil, err
	}

	for _, path := range searchPaths {
		pkgDir := basePath + "/" + path

		if util.NodeNotExist(pkgDir) {
			continue
		}

		dirList, err := repo.FilteredSearchList(path)
		if err != nil {
			return nil, util.NewNewtError(err.Error())
		}

		for _, subDir := range dirList {
			if err := ReadLocalPackageRecursive(repo, pkgList, basePath,
				filepath.Join(path, subDir)); err != nil {
				return nil, util.NewNewtError(err.Error())
			}
		}
	}

	return &pkgList, nil
}
<commit_msg>newt; error was not propagated properly when loading packages<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package pkg

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	log "github.com/Sirupsen/logrus"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/newt/newtutil"
	"mynewt.apache.org/newt/newt/repo"
	"mynewt.apache.org/newt/util"
	"mynewt.apache.org/newt/viper"
	"mynewt.apache.org/newt/yaml"
)

var PackageHashIgnoreDirs = map[string]bool{
	"obj": true,
	"bin": true,
	".":   true,
}

var LocalPackageSpecialNames = map[string]bool{
	"src":     true,
	"include": true,
	"bin":     true,
}

type LocalPackage struct {
	repo        *repo.Repo
	name        string
	basePath    string
	packageType interfaces.PackageType

	// General information about the package
	desc *PackageDesc
	// Dependencies for this package
	deps []*Dependency

	// Package init function name and stage.  These are used to generate the
	// sysinit C file.
	initFnName string
	initStage  int

	// Extra package-specific settings that don't come from syscfg.  For
	// example, SELFTEST gets set when the newt test command is used.
	injectedSettings map[string]string

	// Pointer to pkg.yml configuration structure
	Viper *viper.Viper

	// Names of all source yml files; used to determine if rebuild required.
	cfgFilenames []string
}

func NewLocalPackage(r *repo.Repo, pkgDir string) *LocalPackage {
	pkg := &LocalPackage{
		desc: &PackageDesc{},
		// XXX: Initialize viper object; clients should not need to check for
		// nil pointer.
		repo:             r,
		basePath:         filepath.Clean(pkgDir) + "/", // XXX: Remove slash.
		injectedSettings: map[string]string{},
	}
	return pkg
}

func (pkg *LocalPackage) Name() string {
	return pkg.name
}

func (pkg *LocalPackage) FullName() string {
	r := pkg.Repo()
	if r.IsLocal() {
		return pkg.Name()
	} else {
		return newtutil.BuildPackageString(r.Name(), pkg.Name())
	}
}

func (pkg *LocalPackage) BasePath() string {
	return filepath.Clean(pkg.basePath)
}

func (pkg *LocalPackage) Type() interfaces.PackageType {
	return pkg.packageType
}

func (pkg *LocalPackage) Repo() interfaces.RepoInterface {
	return pkg.repo
}

func (pkg *LocalPackage) Desc() *PackageDesc {
	return pkg.desc
}

func (pkg *LocalPackage) SetName(name string) {
	pkg.name = name
	// XXX: Also set "pkg.name" in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetBasePath(basePath string) {
	pkg.basePath = basePath
}

func (pkg *LocalPackage) SetType(packageType interfaces.PackageType) {
	pkg.packageType = packageType
	// XXX: Also set "pkg.type" in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetDesc(desc *PackageDesc) {
	pkg.desc = desc
	// XXX: Also set desc fields in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetRepo(r *repo.Repo) {
	pkg.repo = r
}

func (pkg *LocalPackage) Hash() (string, error) {
	hash := sha1.New()

	err := filepath.Walk(pkg.basePath,
		func(path string, info os.FileInfo, err error) error {
			name := info.Name()
			if PackageHashIgnoreDirs[name] {
				return filepath.SkipDir
			}

			if info.IsDir() {
				// SHA the directory name into the hash
				hash.Write([]byte(name))
			} else {
				// SHA the file name & contents into the hash
				contents, err := ioutil.ReadFile(path)
				if err != nil {
					return err
				}
				hash.Write(contents)
			}
			return nil
		})
	if err != nil && err != filepath.SkipDir {
		return "", util.NewNewtError(err.Error())
	}

	hashStr := fmt.Sprintf("%x", hash.Sum(nil))

	return hashStr, nil
}

func (pkg *LocalPackage) CfgFilenames() []string {
	return pkg.cfgFilenames
}

func (pkg *LocalPackage) AddCfgFilename(cfgFilename string) {
	pkg.cfgFilenames = append(pkg.cfgFilenames, cfgFilename)
}

func (pkg *LocalPackage) HasDep(searchDep *Dependency) bool {
	for _, dep := range pkg.deps {
		if dep.String() == searchDep.String() {
			return true
		}
	}
	return false
}

func (pkg *LocalPackage) AddDep(dep *Dependency) {
	// Remember the name of the configuration file so that it can be specified
	// as a dependency to the compiler.
	pkg.deps = append(pkg.deps, dep)
}

func (pkg *LocalPackage) Deps() []*Dependency {
	return pkg.deps
}

func (pkg *LocalPackage) readDesc(v *viper.Viper) (*PackageDesc, error) {
	pdesc := &PackageDesc{}

	pdesc.Author = v.GetString("pkg.author")
	pdesc.Homepage = v.GetString("pkg.homepage")
	pdesc.Description = v.GetString("pkg.description")
	pdesc.Keywords = v.GetStringSlice("pkg.keywords")

	return pdesc, nil
}

func (pkg *LocalPackage) sequenceString(key string) string {
	var buffer bytes.Buffer

	if pkg.Viper != nil {
		for _, f := range pkg.Viper.GetStringSlice(key) {
			buffer.WriteString("    - " + yaml.EscapeString(f) + "\n")
		}
	}

	if buffer.Len() == 0 {
		return ""
	} else {
		return key + ":\n" + buffer.String()
	}
}

// Saves the package's pkg.yml file.
// NOTE: This does not save every field in the package.  Only the fields
// necessary for creating a new target get saved.
func (pkg *LocalPackage) Save() error {
	dirpath := pkg.BasePath()
	if err := os.MkdirAll(dirpath, 0755); err != nil {
		return util.NewNewtError(err.Error())
	}

	filepath := dirpath + "/" + PACKAGE_FILE_NAME
	file, err := os.Create(filepath)
	if err != nil {
		return util.NewNewtError(err.Error())
	}
	defer file.Close()

	file.WriteString("### Package: " + pkg.Name() + "\n")

	// XXX: Just iterate viper object's settings rather than calling out
	// cached settings individually.
	file.WriteString("pkg.name: " + yaml.EscapeString(pkg.Name()) + "\n")
	file.WriteString("pkg.type: " +
		yaml.EscapeString(PackageTypeNames[pkg.Type()]) + "\n")
	file.WriteString("pkg.description: " +
		yaml.EscapeString(pkg.Desc().Description) + "\n")
	file.WriteString("pkg.author: " +
		yaml.EscapeString(pkg.Desc().Author) + "\n")
	file.WriteString("pkg.homepage: " +
		yaml.EscapeString(pkg.Desc().Homepage) + "\n")

	file.WriteString("\n")

	file.WriteString(pkg.sequenceString("pkg.aflags"))
	file.WriteString(pkg.sequenceString("pkg.cflags"))
	file.WriteString(pkg.sequenceString("pkg.features"))
	file.WriteString(pkg.sequenceString("pkg.lflags"))

	return nil
}

// Load reads everything that isn't identity specific into the
// package
func (pkg *LocalPackage) Load() error {
	// Load configuration
	log.Debugf("Loading configuration for package %s", pkg.basePath)

	v, err := util.ReadConfig(pkg.basePath,
		strings.TrimSuffix(PACKAGE_FILE_NAME, ".yml"))
	if err != nil {
		return err
	}
	pkg.Viper = v

	// Set package name from the package
	pkg.name = v.GetString("pkg.name")

	typeString := v.GetString("pkg.type")
	pkg.packageType = PACKAGE_TYPE_LIB
	for t, n := range PackageTypeNames {
		if typeString == n {
			pkg.packageType = t
			break
		}
	}

	pkg.initFnName = v.GetString("pkg.init_function")
	pkg.initStage = v.GetInt("pkg.init_stage")

	// Read the package description from the file
	pkg.desc, err = pkg.readDesc(v)
	if err != nil {
		return err
	}

	pkg.AddCfgFilename(pkg.basePath + PACKAGE_FILE_NAME)

	return nil
}

func (pkg *LocalPackage) InitStage() int {
	return pkg.initStage
}

func (pkg *LocalPackage) InitFnName() string {
	return pkg.initFnName
}

func (pkg *LocalPackage) InjectedSettings() map[string]string {
	return pkg.injectedSettings
}

func (pkg *LocalPackage) Clone(newRepo *repo.Repo,
	newName string) *LocalPackage {

	// XXX: Validate name.

	// Copy the package.
	newPkg := *pkg
	newPkg.repo = newRepo
	newPkg.name = newName
	newPkg.basePath = newRepo.Path() + "/" + newPkg.name

	// Insert the clone into the global package map.
	proj := interfaces.GetProject()
	pMap := proj.PackageList()
	(*pMap[newRepo.Name()])[newPkg.name] = &newPkg

	return &newPkg
}

func LoadLocalPackage(repo *repo.Repo, pkgDir string) (*LocalPackage, error) {
	pkg := NewLocalPackage(repo, pkgDir)
	err := pkg.Load()
	return pkg, err
}

func LocalPackageSpecialName(dirName string) bool {
	_, ok := LocalPackageSpecialNames[dirName]
	return ok
}

func ReadLocalPackageRecursive(repo *repo.Repo,
	pkgList map[string]interfaces.PackageInterface, basePath string,
	pkgName string) error {

	dirList, err := repo.FilteredSearchList(pkgName)
	if err != nil {
		return util.NewNewtError(err.Error())
	}

	for _, name := range dirList {
		if LocalPackageSpecialName(name) || strings.HasPrefix(name, ".") {
			continue
		}

		if err := ReadLocalPackageRecursive(repo, pkgList, basePath,
			filepath.Join(pkgName, name)); err != nil {
			return err
		}
	}

	if util.NodeNotExist(filepath.Join(basePath, pkgName, PACKAGE_FILE_NAME)) {
		return nil
	}

	pkg, err := LoadLocalPackage(repo, filepath.Join(basePath, pkgName))
	if err != nil {
		return err
	}

	if oldPkg, ok := pkgList[pkg.Name()]; ok {
		oldlPkg := oldPkg.(*LocalPackage)
		return util.FmtNewtError("Multiple packages with same pkg.name=%s "+
			"in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(),
			oldlPkg.BasePath(), pkg.BasePath())
	}

	pkgList[pkg.Name()] = pkg

	return nil
}

func ReadLocalPackages(repo *repo.Repo,
	basePath string) (*map[string]interfaces.PackageInterface, error) {

	pkgList := map[string]interfaces.PackageInterface{}

	searchPaths, err := repo.FilteredSearchList("")
	if err != nil {
		return nil, err
	}

	for _, path := range searchPaths {
		pkgDir := basePath + "/" + path

		if util.NodeNotExist(pkgDir) {
			continue
		}

		dirList, err := repo.FilteredSearchList(path)
		if err != nil {
			return nil, util.NewNewtError(err.Error())
		}

		for _, subDir := range dirList {
			if err := ReadLocalPackageRecursive(repo, pkgList, basePath,
				filepath.Join(path, subDir)); err != nil {
				return nil, err
			}
		}
	}

	return &pkgList, nil
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package pkg

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/newt/newtutil"
	"mynewt.apache.org/newt/newt/repo"
	"mynewt.apache.org/newt/util"
	"mynewt.apache.org/newt/viper"
	"mynewt.apache.org/newt/yaml"
)

var PackageHashIgnoreDirs = map[string]bool{
	"obj": true,
	"bin": true,
	".":   true,
}

var LocalPackageSpecialNames = map[string]bool{
	"src":     true,
	"include": true,
	"bin":     true,
}

type LocalPackage struct {
	repo        *repo.Repo
	name        string
	basePath    string
	packageType interfaces.PackageType

	// General information about the package
	desc *PackageDesc

	// Package init function name and stage.  These are used to generate the
	// sysinit C file.
	init map[string]int

	// Extra package-specific settings that don't come from syscfg.  For
	// example, SELFTEST gets set when the newt test command is used.
	injectedSettings map[string]string

	// Settings read from pkg.yml.
	PkgV *viper.Viper

	// Settings read from syscfg.yml.
	SyscfgV *viper.Viper

	// Names of all source yml files; used to determine if rebuild required.
	cfgFilenames []string
}

func NewLocalPackage(r *repo.Repo, pkgDir string) *LocalPackage {
	pkg := &LocalPackage{
		desc:             &PackageDesc{},
		PkgV:             viper.New(),
		SyscfgV:          viper.New(),
		repo:             r,
		basePath:         filepath.ToSlash(filepath.Clean(pkgDir)),
		init:             map[string]int{},
		injectedSettings: map[string]string{},
	}
	return pkg
}

func (pkg *LocalPackage) Name() string {
	return pkg.name
}

func (pkg *LocalPackage) FullName() string {
	r := pkg.Repo()
	if r.IsLocal() {
		return pkg.Name()
	} else {
		return newtutil.BuildPackageString(r.Name(), pkg.Name())
	}
}

func (pkg *LocalPackage) BasePath() string {
	return pkg.basePath
}

func (pkg *LocalPackage) RelativePath() string {
	proj := interfaces.GetProject()
	return strings.TrimPrefix(pkg.BasePath(), proj.Path())
}

func (pkg *LocalPackage) Type() interfaces.PackageType {
	return pkg.packageType
}

func (pkg *LocalPackage) Repo() interfaces.RepoInterface {
	return pkg.repo
}

func (pkg *LocalPackage) Desc() *PackageDesc {
	return pkg.desc
}

func (pkg *LocalPackage) SetName(name string) {
	pkg.name = name
	// XXX: Also set "pkg.name" in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetBasePath(basePath string) {
	pkg.basePath = filepath.ToSlash(filepath.Clean(basePath))
}

func (pkg *LocalPackage) SetType(packageType interfaces.PackageType) {
	pkg.packageType = packageType
	// XXX: Also set "pkg.type" in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetDesc(desc *PackageDesc) {
	pkg.desc = desc
	// XXX: Also set desc fields in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetRepo(r *repo.Repo) {
	pkg.repo = r
}

func (pkg *LocalPackage) Hash() (string, error) {
	hash := sha1.New()

	err := filepath.Walk(pkg.basePath,
		func(path string, info os.FileInfo, err error) error {
			name := info.Name()
			if PackageHashIgnoreDirs[name] {
				return filepath.SkipDir
			}

			if info.IsDir() {
				// SHA the directory name into the hash
				hash.Write([]byte(name))
			} else {
				// SHA the file name & contents into the hash
				contents, err := ioutil.ReadFile(path)
				if err != nil {
					return err
				}
				hash.Write(contents)
			}
			return nil
		})
	if err != nil && err != filepath.SkipDir {
		return "", util.NewNewtError(err.Error())
	}

	hashStr := fmt.Sprintf("%x", hash.Sum(nil))

	return hashStr, nil
}

func (pkg *LocalPackage) CfgFilenames() []string {
	return pkg.cfgFilenames
}

func (pkg *LocalPackage) AddCfgFilename(cfgFilename string) {
	pkg.cfgFilenames = append(pkg.cfgFilenames, cfgFilename)
}

func (pkg *LocalPackage) readDesc(v *viper.Viper) (*PackageDesc, error) {
	pdesc := &PackageDesc{}

	pdesc.Author = v.GetString("pkg.author")
	pdesc.Homepage = v.GetString("pkg.homepage")
	pdesc.Description = v.GetString("pkg.description")
	pdesc.Keywords = v.GetStringSlice("pkg.keywords")

	return pdesc, nil
}

func (pkg *LocalPackage) sequenceString(key string) string {
	var buffer bytes.Buffer

	if pkg.PkgV != nil {
		for _, f := range pkg.PkgV.GetStringSlice(key) {
			buffer.WriteString("    - " + yaml.EscapeString(f) + "\n")
		}
	}

	if buffer.Len() == 0 {
		return ""
	} else {
		return key + ":\n" + buffer.String()
	}
}

func (lpkg *LocalPackage) SaveSyscfgVals() error {
	dirpath := lpkg.BasePath()
	if err := os.MkdirAll(dirpath, 0755); err != nil {
		return util.NewNewtError(err.Error())
	}

	filepath := dirpath + "/" + SYSCFG_YAML_FILENAME

	syscfgVals := lpkg.SyscfgV.GetStringMapString("syscfg.vals")
	if syscfgVals == nil || len(syscfgVals) == 0 {
		os.Remove(filepath)
		return nil
	}

	file, err := os.Create(filepath)
	if err != nil {
		return util.NewNewtError(err.Error())
	}
	defer file.Close()

	names := make([]string, 0, len(syscfgVals))
	for k, _ := range syscfgVals {
		names = append(names, k)
	}
	sort.Strings(names)

	fmt.Fprintf(file, "### Package: %s\n", lpkg.Name())
	fmt.Fprintf(file, "\n")
	fmt.Fprintf(file, "syscfg.vals:\n")
	for _, name := range names {
		fmt.Fprintf(file, "    %s: %s\n", name, yaml.EscapeString(syscfgVals[name]))
	}

	return nil
}

// Saves the package's pkg.yml file.
// NOTE: This does not save every field in the package.  Only the fields
// necessary for creating a new target get saved.
func (pkg *LocalPackage) Save() error {
	dirpath := pkg.BasePath()
	if err := os.MkdirAll(dirpath, 0755); err != nil {
		return util.NewNewtError(err.Error())
	}

	filepath := dirpath + "/" + PACKAGE_FILE_NAME
	file, err := os.Create(filepath)
	if err != nil {
		return util.NewNewtError(err.Error())
	}
	defer file.Close()

	file.WriteString("### Package: " + pkg.Name() + "\n")

	// XXX: Just iterate viper object's settings rather than calling out
	// cached settings individually.
	file.WriteString("pkg.name: " + yaml.EscapeString(pkg.Name()) + "\n")
	file.WriteString("pkg.type: " +
		yaml.EscapeString(PackageTypeNames[pkg.Type()]) + "\n")
	file.WriteString("pkg.description: " +
		yaml.EscapeString(pkg.Desc().Description) + "\n")
	file.WriteString("pkg.author: " +
		yaml.EscapeString(pkg.Desc().Author) + "\n")
	file.WriteString("pkg.homepage: " +
		yaml.EscapeString(pkg.Desc().Homepage) + "\n")

	file.WriteString("\n")

	file.WriteString(pkg.sequenceString("pkg.aflags"))
	file.WriteString(pkg.sequenceString("pkg.cflags"))
	file.WriteString(pkg.sequenceString("pkg.lflags"))

	return nil
}

// Load reads everything that isn't identity specific into the
// package
func (pkg *LocalPackage) Load() error {
	// Load configuration
	log.Debugf("Loading configuration for package %s", pkg.basePath)

	var err error

	pkg.PkgV, err = util.ReadConfig(pkg.basePath,
		strings.TrimSuffix(PACKAGE_FILE_NAME, ".yml"))
	if err != nil {
		return err
	}
	pkg.AddCfgFilename(pkg.basePath + "/" + PACKAGE_FILE_NAME)

	// Set package name from the package
	pkg.name = pkg.PkgV.GetString("pkg.name")

	typeString := pkg.PkgV.GetString("pkg.type")
	pkg.packageType = PACKAGE_TYPE_LIB
	for t, n := range PackageTypeNames {
		if typeString == n {
			pkg.packageType = t
			break
		}
	}

	init := pkg.PkgV.GetStringMapString("pkg.init")
	for name, stageStr := range init {
		stage, err := strconv.ParseInt(stageStr, 10, 64)
		if err != nil {
			return util.NewNewtError(fmt.Sprintf("Parsing pkg %s config: %s",
				pkg.FullName(), err.Error()))
		}
		pkg.init[name] = int(stage)
	}
	initFnName := pkg.PkgV.GetString("pkg.init_function")
	initStage := pkg.PkgV.GetInt("pkg.init_stage")

	if initFnName != "" {
		pkg.init[initFnName] = initStage
	}

	// Read the package description from the file
	pkg.desc, err = pkg.readDesc(pkg.PkgV)
	if err != nil {
		return err
	}

	// Load syscfg settings.
	if util.NodeExist(pkg.basePath + "/" + SYSCFG_YAML_FILENAME) {
		pkg.SyscfgV, err = util.ReadConfig(pkg.basePath,
			strings.TrimSuffix(SYSCFG_YAML_FILENAME, ".yml"))
		if err != nil {
			return err
		}
		pkg.AddCfgFilename(pkg.basePath + "/" + SYSCFG_YAML_FILENAME)
	}

	return nil
}

func (pkg *LocalPackage) Init() map[string]int {
	return pkg.init
}

func (pkg *LocalPackage) InjectedSettings() map[string]string {
	return pkg.injectedSettings
}

func (pkg *LocalPackage) Clone(newRepo *repo.Repo,
	newName string) *LocalPackage {

	// XXX: Validate name.

	// Copy the package.
	newPkg := *pkg
	newPkg.repo = newRepo
	newPkg.name = newName
	newPkg.basePath = newRepo.Path() + "/" + newPkg.name

	// Insert the clone into the global package map.
	proj := interfaces.GetProject()
	pMap := proj.PackageList()

	(*pMap[newRepo.Name()])[newPkg.name] = &newPkg

	return &newPkg
}

func LoadLocalPackage(repo *repo.Repo, pkgDir string) (*LocalPackage, error) {
	pkg := NewLocalPackage(repo, pkgDir)
	err := pkg.Load()
	return pkg, err
}

func LocalPackageSpecialName(dirName string) bool {
	_, ok := LocalPackageSpecialNames[dirName]
	return ok
}

func ReadLocalPackageRecursive(repo *repo.Repo,
	pkgList map[string]interfaces.PackageInterface, basePath string,
	pkgName string, searchedMap map[string]struct{}) ([]string, error) {

	var warnings []string

	dirList, err := repo.FilteredSearchList(pkgName, searchedMap)
	if err != nil {
		return warnings, util.NewNewtError(err.Error())
	}

	for _, name := range dirList {
		if LocalPackageSpecialName(name) || strings.HasPrefix(name, ".") {
			continue
		}

		subWarnings, err := ReadLocalPackageRecursive(repo, pkgList,
			basePath, filepath.Join(pkgName, name), searchedMap)
		warnings = append(warnings, subWarnings...)
		if err != nil {
			return warnings, err
		}
	}

	if util.NodeNotExist(filepath.Join(basePath, pkgName,
		PACKAGE_FILE_NAME)) {

		return warnings, nil
	}

	pkg, err := LoadLocalPackage(repo, filepath.Join(basePath, pkgName))
	if err != nil {
		warnings = append(warnings, err.Error())
		return warnings, nil
	}

	if oldPkg, ok := pkgList[pkg.Name()]; ok {
		oldlPkg := oldPkg.(*LocalPackage)
		warnings = append(warnings,
			fmt.Sprintf("Multiple packages with same pkg.name=%s "+
				"in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(),
				oldlPkg.BasePath(), pkg.BasePath()))

		return warnings, nil
	}

	pkgList[pkg.Name()] = pkg

	return warnings, nil
}

func ReadLocalPackages(repo *repo.Repo, basePath string) (
	*map[string]interfaces.PackageInterface, []string, error) {

	pkgMap := &map[string]interfaces.PackageInterface{}

	// Keep track of which directories we have traversed.  Prevent infinite
	// loops caused by symlink cycles by not inspecting the same directory
	// twice.
	searchedMap := map[string]struct{}{}

	warnings, err := ReadLocalPackageRecursive(repo, *pkgMap,
		basePath, "", searchedMap)

	return pkgMap, warnings, err
}
<commit_msg>MYNEWT-860 Newt - Empty `pkg.yml` issues.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package pkg

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/newt/newtutil"
	"mynewt.apache.org/newt/newt/repo"
	"mynewt.apache.org/newt/util"
	"mynewt.apache.org/newt/viper"
	"mynewt.apache.org/newt/yaml"
)

var PackageHashIgnoreDirs = map[string]bool{
	"obj": true,
	"bin": true,
	".":   true,
}

var LocalPackageSpecialNames = map[string]bool{
	"src":     true,
	"include": true,
	"bin":     true,
}

type LocalPackage struct {
	repo        *repo.Repo
	name        string
	basePath    string
	packageType interfaces.PackageType

	// General information about the package
	desc *PackageDesc

	// Package init function name and stage.  These are used to generate the
	// sysinit C file.
	init map[string]int

	// Extra package-specific settings that don't come from syscfg.  For
	// example, SELFTEST gets set when the newt test command is used.
	injectedSettings map[string]string

	// Settings read from pkg.yml.
	PkgV *viper.Viper

	// Settings read from syscfg.yml.
	SyscfgV *viper.Viper

	// Names of all source yml files; used to determine if rebuild required.
	cfgFilenames []string
}

func NewLocalPackage(r *repo.Repo, pkgDir string) *LocalPackage {
	pkg := &LocalPackage{
		desc:             &PackageDesc{},
		PkgV:             viper.New(),
		SyscfgV:          viper.New(),
		repo:             r,
		basePath:         filepath.ToSlash(filepath.Clean(pkgDir)),
		init:             map[string]int{},
		injectedSettings: map[string]string{},
	}
	return pkg
}

func (pkg *LocalPackage) Name() string {
	return pkg.name
}

func (pkg *LocalPackage) FullName() string {
	r := pkg.Repo()
	if r.IsLocal() {
		return pkg.Name()
	} else {
		return newtutil.BuildPackageString(r.Name(), pkg.Name())
	}
}

func (pkg *LocalPackage) BasePath() string {
	return pkg.basePath
}

func (pkg *LocalPackage) RelativePath() string {
	proj := interfaces.GetProject()
	return strings.TrimPrefix(pkg.BasePath(), proj.Path())
}

func (pkg *LocalPackage) Type() interfaces.PackageType {
	return pkg.packageType
}

func (pkg *LocalPackage) Repo() interfaces.RepoInterface {
	return pkg.repo
}

func (pkg *LocalPackage) Desc() *PackageDesc {
	return pkg.desc
}

func (pkg *LocalPackage) SetName(name string) {
	pkg.name = name
	// XXX: Also set "pkg.name" in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetBasePath(basePath string) {
	pkg.basePath = filepath.ToSlash(filepath.Clean(basePath))
}

func (pkg *LocalPackage) SetType(packageType interfaces.PackageType) {
	pkg.packageType = packageType
	// XXX: Also set "pkg.type" in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetDesc(desc *PackageDesc) {
	pkg.desc = desc
	// XXX: Also set desc fields in viper object (possibly just remove cached
	// variable from code entirely).
}

func (pkg *LocalPackage) SetRepo(r *repo.Repo) {
	pkg.repo = r
}

func (pkg *LocalPackage) Hash() (string, error) {
	hash := sha1.New()

	err := filepath.Walk(pkg.basePath,
		func(path string, info os.FileInfo, err error) error {
			name := info.Name()
			if PackageHashIgnoreDirs[name] {
				return filepath.SkipDir
			}

			if info.IsDir() {
				// SHA the directory name into the hash
				hash.Write([]byte(name))
			} else {
				// SHA the file name & contents into the hash
				contents, err := ioutil.ReadFile(path)
				if err != nil {
					return err
				}
				hash.Write(contents)
			}
			return nil
		})
	if err != nil && err != filepath.SkipDir {
		return "", util.NewNewtError(err.Error())
	}

	hashStr := fmt.Sprintf("%x", hash.Sum(nil))

	return hashStr, nil
}

func (pkg *LocalPackage) CfgFilenames() []string {
	return pkg.cfgFilenames
}

func (pkg *LocalPackage) AddCfgFilename(cfgFilename string) {
	pkg.cfgFilenames = append(pkg.cfgFilenames, cfgFilename)
}

func (pkg *LocalPackage) readDesc(v *viper.Viper) (*PackageDesc, error) {
	pdesc := &PackageDesc{}

	pdesc.Author = v.GetString("pkg.author")
	pdesc.Homepage = v.GetString("pkg.homepage")
	pdesc.Description = v.GetString("pkg.description")
	pdesc.Keywords = v.GetStringSlice("pkg.keywords")

	return pdesc, nil
}

func (pkg *LocalPackage) sequenceString(key string) string {
	var buffer bytes.Buffer

	if pkg.PkgV != nil {
		for _, f := range pkg.PkgV.GetStringSlice(key) {
			buffer.WriteString("    - " + yaml.EscapeString(f) + "\n")
		}
	}

	if buffer.Len() == 0 {
		return ""
	} else {
		return key + ":\n" + buffer.String()
	}
}

func (lpkg *LocalPackage) SaveSyscfgVals() error {
	dirpath := lpkg.BasePath()
	if err := os.MkdirAll(dirpath, 0755); err != nil {
		return util.NewNewtError(err.Error())
	}

	filepath := dirpath + "/" + SYSCFG_YAML_FILENAME

	syscfgVals := lpkg.SyscfgV.GetStringMapString("syscfg.vals")
	if syscfgVals == nil || len(syscfgVals) == 0 {
		os.Remove(filepath)
		return nil
	}

	file, err := os.Create(filepath)
	if err != nil {
		return util.NewNewtError(err.Error())
	}
	defer file.Close()

	names := make([]string, 0, len(syscfgVals))
	for k, _ := range syscfgVals {
		names = append(names, k)
	}
	sort.Strings(names)

	fmt.Fprintf(file, "### Package: %s\n", lpkg.Name())
	fmt.Fprintf(file, "\n")
	fmt.Fprintf(file, "syscfg.vals:\n")
	for _, name := range names {
		fmt.Fprintf(file, "    %s: %s\n", name, yaml.EscapeString(syscfgVals[name]))
	}

	return nil
}

// Saves the package's pkg.yml file.
// NOTE: This does not save every field in the package.  Only the fields
// necessary for creating a new target get saved.
func (pkg *LocalPackage) Save() error {
	dirpath := pkg.BasePath()
	if err := os.MkdirAll(dirpath, 0755); err != nil {
		return util.NewNewtError(err.Error())
	}

	filepath := dirpath + "/" + PACKAGE_FILE_NAME
	file, err := os.Create(filepath)
	if err != nil {
		return util.NewNewtError(err.Error())
	}
	defer file.Close()

	file.WriteString("### Package: " + pkg.Name() + "\n")

	// XXX: Just iterate viper object's settings rather than calling out
	// cached settings individually.
	file.WriteString("pkg.name: " + yaml.EscapeString(pkg.Name()) + "\n")
	file.WriteString("pkg.type: " +
		yaml.EscapeString(PackageTypeNames[pkg.Type()]) + "\n")
	file.WriteString("pkg.description: " +
		yaml.EscapeString(pkg.Desc().Description) + "\n")
	file.WriteString("pkg.author: " +
		yaml.EscapeString(pkg.Desc().Author) + "\n")
	file.WriteString("pkg.homepage: " +
		yaml.EscapeString(pkg.Desc().Homepage) + "\n")

	file.WriteString("\n")

	file.WriteString(pkg.sequenceString("pkg.aflags"))
	file.WriteString(pkg.sequenceString("pkg.cflags"))
	file.WriteString(pkg.sequenceString("pkg.lflags"))

	return nil
}

// Load reads everything that isn't identity specific into the package
func (pkg *LocalPackage) Load() error {
	// Load configuration
	log.Debugf("Loading configuration for package %s", pkg.basePath)

	var err error

	pkg.PkgV, err = util.ReadConfig(pkg.basePath,
		strings.TrimSuffix(PACKAGE_FILE_NAME, ".yml"))
	if err != nil {
		return err
	}
	pkg.AddCfgFilename(pkg.basePath + "/" + PACKAGE_FILE_NAME)

	// Set package name from the package
	pkg.name = pkg.PkgV.GetString("pkg.name")
	if pkg.name == "" {
		return util.FmtNewtError(
			"Package \"%s\" missing \"pkg.name\" field in its `pkg.yml` file",
			pkg.basePath)
	}

	typeString := pkg.PkgV.GetString("pkg.type")
	pkg.packageType = PACKAGE_TYPE_LIB
	for t, n := range PackageTypeNames {
		if typeString == n {
			pkg.packageType = t
			break
		}
	}

	init := pkg.PkgV.GetStringMapString("pkg.init")
	for name, stageStr := range init {
		stage, err := strconv.ParseInt(stageStr, 10, 64)
		if err != nil {
			return util.NewNewtError(fmt.Sprintf("Parsing pkg %s config: %s",
				pkg.FullName(), err.Error()))
		}
		pkg.init[name] = int(stage)
	}
	initFnName := pkg.PkgV.GetString("pkg.init_function")
	initStage := pkg.PkgV.GetInt("pkg.init_stage")

	if initFnName != "" {
		pkg.init[initFnName] = initStage
	}

	// Read the package description from the file
	pkg.desc, err = pkg.readDesc(pkg.PkgV)
	if err != nil {
		return err
	}

	// Load syscfg settings.
	if util.NodeExist(pkg.basePath + "/" + SYSCFG_YAML_FILENAME) {
		pkg.SyscfgV, err = util.ReadConfig(pkg.basePath,
			strings.TrimSuffix(SYSCFG_YAML_FILENAME, ".yml"))
		if err != nil {
			return err
		}
		pkg.AddCfgFilename(pkg.basePath + "/" + SYSCFG_YAML_FILENAME)
	}

	return nil
}

func (pkg *LocalPackage) Init() map[string]int {
	return pkg.init
}

func (pkg *LocalPackage) InjectedSettings() map[string]string {
	return pkg.injectedSettings
}

func (pkg *LocalPackage) Clone(newRepo *repo.Repo,
	newName string) *LocalPackage {

	// XXX: Validate name.

	// Copy the package.
	newPkg := *pkg
	newPkg.repo = newRepo
	newPkg.name = newName
	newPkg.basePath = newRepo.Path() + "/" + newPkg.name

	// Insert the clone into the global package map.
	proj := interfaces.GetProject()
	pMap := proj.PackageList()

	(*pMap[newRepo.Name()])[newPkg.name] = &newPkg

	return &newPkg
}

func LoadLocalPackage(repo *repo.Repo, pkgDir string) (*LocalPackage, error) {
	pkg := NewLocalPackage(repo, pkgDir)
	err := pkg.Load()
	if err != nil {
		err = util.FmtNewtError("%s; ignoring package.", err.Error())
		return nil, err
	}

	return pkg, err
}

func LocalPackageSpecialName(dirName string) bool {
	_, ok := LocalPackageSpecialNames[dirName]
	return ok
}

func ReadLocalPackageRecursive(repo *repo.Repo,
	pkgList map[string]interfaces.PackageInterface, basePath string,
	pkgName string, searchedMap map[string]struct{}) ([]string, error) {

	var warnings []string

	dirList, err := repo.FilteredSearchList(pkgName, searchedMap)
	if err != nil {
		return warnings, util.NewNewtError(err.Error())
	}

	for _, name := range dirList {
		if LocalPackageSpecialName(name) || strings.HasPrefix(name, ".") {
			continue
		}

		subWarnings, err := ReadLocalPackageRecursive(repo, pkgList,
			basePath, filepath.Join(pkgName, name), searchedMap)
		warnings = append(warnings, subWarnings...)
		if err != nil {
			return warnings, err
		}
	}

	if util.NodeNotExist(filepath.Join(basePath, pkgName,
		PACKAGE_FILE_NAME)) {

		return warnings, nil
	}

	pkg, err := LoadLocalPackage(repo, filepath.Join(basePath, pkgName))
	if err != nil {
		warnings = append(warnings, err.Error())
		return warnings, nil
	}

	if oldPkg, ok := pkgList[pkg.Name()]; ok {
		oldlPkg := oldPkg.(*LocalPackage)
		warnings = append(warnings,
			fmt.Sprintf("Multiple packages with same pkg.name=%s "+
				"in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(),
				oldlPkg.BasePath(), pkg.BasePath()))

		return warnings, nil
	}

	pkgList[pkg.Name()] = pkg

	return warnings, nil
}

func ReadLocalPackages(repo *repo.Repo, basePath string) (
	*map[string]interfaces.PackageInterface, []string, error) {

	pkgMap := &map[string]interfaces.PackageInterface{}

	// Keep track of which directories we have traversed.  Prevent infinite
	// loops caused by symlink cycles by not inspecting the same directory
	// twice.
	searchedMap := map[string]struct{}{}

	warnings, err := ReadLocalPackageRecursive(repo, *pkgMap,
		basePath, "", searchedMap)

	return pkgMap, warnings, err
}
<|endoftext|>


<commit_before>package main

const ninjaLaunchShim = `#!/bin/bash

export sphere_installDirectory=/apps/ninjasphere/current
export PATH=/apps/ninjasphere/current/bin/:$SNAPP_APP_PATH/bin:$PATH

exec "$@"`

const ninjaAppProfileRediculouslyPermissive = `# vim:syntax=apparmor

#include <tunables/global>

# Specified profile variables
###VAR###

###PROFILEATTACH### (attach_disconnected) {
  #include <abstractions/base>
  #include <abstractions/consoles>
  #include <abstractions/openssl>

  # for python apps/services
  #include <abstractions/python>
  /usr/bin/python{,2,2.[0-9]*,3,3.[0-9]*} ixr,

  # for perl apps/services
  #include <abstractions/perl>
  /usr/bin/perl{,5*} ixr,

  # for bash 'binaries' (do *not* use abstractions/bash)
  # user-specific bash files
  /bin/bash ixr,
  /bin/dash ixr,
  /etc/bash.bashrc r,
  /usr/share/terminfo/** r,
  /etc/inputrc r,
  deny @{HOME}/.inputrc r,
  # Common utilities for shell scripts
  /{,usr/}bin/{,g,m}awk ixr,
  /{,usr/}bin/basename ixr,
  /{,usr/}bin/bunzip2 ixr,
  /{,usr/}bin/bzcat ixr,
  /{,usr/}bin/bzdiff ixr,
  /{,usr/}bin/bzgrep ixr,
  /{,usr/}bin/bzip2 ixr,
  /{,usr/}bin/cat ixr,
  /{,usr/}bin/chmod ixr,
  /{,usr/}bin/cmp ixr,
  /{,usr/}bin/cp ixr,
  /{,usr/}bin/cpio ixr,
  /{,usr/}bin/cut ixr,
  /{,usr/}bin/date ixr,
  /{,usr/}bin/dd ixr,
  /{,usr/}bin/diff{,3} ixr,
  /{,usr/}bin/dir ixr,
  /{,usr/}bin/dirname ixr,
  /{,usr/}bin/echo ixr,
  /{,usr/}bin/{,e,f,r}grep ixr,
  /{,usr/}bin/env ixr,
  /{,usr/}bin/expr ixr,
  /{,usr/}bin/find ixr,
  /{,usr/}bin/fmt ixr,
  /{,usr/}bin/getopt ixr,
  /{,usr/}bin/false ixr,
  /{,usr/}bin/head ixr,
  /{,usr/}bin/id ixr,
  /{,usr/}bin/igawk ixr,
  /{,usr/}bin/kill ixr,
  /{,usr/}bin/ln ixr,
  /{,usr/}bin/line ixr,
  /{,usr/}bin/link ixr,
  /{,usr/}bin/ls ixr,
  /{,usr/}bin/md5sum ixr,
  /{,usr/}bin/mkdir ixr,
  /{,usr/}bin/mktemp ixr,
  /{,usr/}bin/mv ixr,
  /{,usr/}bin/pgrep ixr,
  /{,usr/}bin/printenv ixr,
  /{,usr/}bin/printf ixr,
  /{,usr/}bin/ps ixr,
  /{,usr/}bin/pwd ixr,
  /{,usr/}bin/readlink ixr,
  /{,usr/}bin/realpath ixr,
  /{,usr/}bin/rev ixr,
  /{,usr/}bin/rm ixr,
  /{,usr/}bin/rmdir ixr,
  /{,usr/}bin/sed ixr,
  /{,usr/}bin/seq ixr,
  /{,usr/}bin/sleep ixr,
  /{,usr/}bin/sort ixr,
  /{,usr/}bin/stat ixr,
  /{,usr/}bin/tac ixr,
  /{,usr/}bin/tail ixr,
  /{,usr/}bin/tar ixr,
  /{,usr/}bin/tee ixr,
  /{,usr/}bin/test ixr,
  /{,usr/}bin/tempfile ixr,
  /{,usr/}bin/touch ixr,
  /{,usr/}bin/tr ixr,
  /{,usr/}bin/true ixr,
  /{,usr/}bin/uname ixr,
  /{,usr/}bin/uniq ixr,
  /{,usr/}bin/unlink ixr,
  /{,usr/}bin/unxz ixr,
  /{,usr/}bin/unzip ixr,
  /{,usr/}bin/vdir ixr,
  /{,usr/}bin/wc ixr,
  /{,usr/}bin/which ixr,
  /{,usr/}bin/xz ixr,
  /{,usr/}bin/yes ixr,
  /{,usr/}bin/zcat ixr,
  /{,usr/}bin/z{,e,f}grep ixr,
  /{,usr/}bin/zip ixr,
  /{,usr/}bin/zipgrep ixr,

  # uptime
  /{,usr/}bin/uptime ixr,
  @{PROC}/uptime r,
  @{PROC}/loadavg r,
  # this is an information leak
  deny /{,var/}run/utmp r,

  # Miscellaneous accesses
  /etc/mime.types r,
  @{PROC}/sys/kernel/hostname r,
  @{PROC}/sys/kernel/osrelease r,

  # Read-only for the install directory
  @{CLICK_DIR}/@{APP_PKGNAME}/                   r,
  @{CLICK_DIR}/@{APP_PKGNAME}/@{APP_VERSION}/    r,
  @{CLICK_DIR}/@{APP_PKGNAME}/@{APP_VERSION}/**  mrklix,

  # Read-only home area for other versions
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/                  r,
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/@{APP_VERSION}/   r,
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/@{APP_VERSION}/** mrkix,

  # Writable home area for this version.
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/@{APP_VERSION}/   w,
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/@{APP_VERSION}/** wl,

  # Read-only system area for other versions
  /var/lib/apps/@{APP_PKGNAME}/   r,
  /var/lib/apps/@{APP_PKGNAME}/** mrkix,

  # TODO: the write on these is needed in case they doesn't exist, but means an
  # app could adjust inode data and affect rollbacks.
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/         w,
  /var/lib/apps/@{APP_PKGNAME}/                  w,

  # Writable system area only for this version
  /var/lib/apps/@{APP_PKGNAME}/@{APP_VERSION}/   w,
  /var/lib/apps/@{APP_PKGNAME}/@{APP_VERSION}/** wl,

  # Writable temp area only for this version (launcher will create this
  # directory on our behalf so only allow readonly on parent)
  /tmp/snapps/@{APP_PKGNAME}/                  r,
  /tmp/snapps/@{APP_PKGNAME}/**                rk,
  /tmp/snapps/@{APP_PKGNAME}/@{APP_VERSION}/   rw,
  /tmp/snapps/@{APP_PKGNAME}/@{APP_VERSION}/** mrwlkix,

  # No abstractions specified

  # Rules specified via policy groups
  # Description: Can access the network
  # Usage: common
  #include <abstractions/nameservice>
  #include <abstractions/ssl_certs>

  @{PROC}/sys/net/core/somaxconn r,

  # We want to explicitly deny access to NetworkManager because its DBus API
  # gives away too much
  deny dbus (receive, send)
       bus=system
       path=/org/freedesktop/NetworkManager,
  deny dbus (receive, send)
       bus=system
       peer=(name=org.freedesktop.NetworkManager),

  # Do the same for ofono (LP: #1226844)
  deny dbus (receive, send)
       bus=system
       interface="org.ofono.Manager",

  # Specified read permissions
  /etc/hosts.allow rk,
  /etc/hosts.deny rk,
  /etc/passwd rk,
  /proc/cmdline rk,
  /sys/bus/i2c/devices/0-0050/eeprom rk,
  /sys/devices/ocp/44e0b000.i2c/i2c-0/0-0050/eeprom rk,
  @{PROC}/ rk,
  @{PROC}/** rk,
  @{PROC}/[0-9]*/stat rk,

  # Specified write permissions
  /sys/bus/i2c/devices/i2c-0/new_device rwk,

  # Ninja
  /{,usr/}bin/xxd ixr,
  /sys/class/net/[a-z0-9]*/address rk,
  /sys/devices/*/*/net/[a-z0-9]*/address rk,
  /proc/cmdline rk,
  /bin/ip ixr,
  /dev/gestic rwk,
  /dev/ttyO3 rwk,
  /dev/tty.ledmatrix rwk,
  /sys/class/gpio rwk,
  /sys/class/gpio/** rwk,
  /sys/devices/virtual/gpio/** rwk,

  # Access back to the framework
  /apps/ninjasphere/*/sphere-schemas/** rk,
  /apps/ninjasphere/*/config/** rk,
  /apps/ninjasphere/*/bin/** ixr,
}`
<commit_msg>Add app and data path to shim.<commit_after>package main

const ninjaLaunchShim = `#!/bin/bash

export sphere_installDirectory=/apps/ninjasphere/current
export PATH=/apps/ninjasphere/current/bin/:$SNAPP_APP_PATH/bin:$PATH

export NINJA_APP_PATH=$SNAPP_APP_PATH
export NINJA_APP_DATA_PATH=$SNAPP_APP_DATA_PATH

exec "$@"`

const ninjaAppProfileRediculouslyPermissive = `# vim:syntax=apparmor

#include <tunables/global>

# Specified profile variables
###VAR###

###PROFILEATTACH### (attach_disconnected) {
  #include <abstractions/base>
  #include <abstractions/consoles>
  #include <abstractions/openssl>

  # for python apps/services
  #include <abstractions/python>
  /usr/bin/python{,2,2.[0-9]*,3,3.[0-9]*} ixr,

  # for perl apps/services
  #include <abstractions/perl>
  /usr/bin/perl{,5*} ixr,

  # for bash 'binaries' (do *not* use abstractions/bash)
  # user-specific bash files
  /bin/bash ixr,
  /bin/dash ixr,
  /etc/bash.bashrc r,
  /usr/share/terminfo/** r,
  /etc/inputrc r,
  deny @{HOME}/.inputrc r,
  # Common utilities for shell scripts
  /{,usr/}bin/{,g,m}awk ixr,
  /{,usr/}bin/basename ixr,
  /{,usr/}bin/bunzip2 ixr,
  /{,usr/}bin/bzcat ixr,
  /{,usr/}bin/bzdiff ixr,
  /{,usr/}bin/bzgrep ixr,
  /{,usr/}bin/bzip2 ixr,
  /{,usr/}bin/cat ixr,
  /{,usr/}bin/chmod ixr,
  /{,usr/}bin/cmp ixr,
  /{,usr/}bin/cp ixr,
  /{,usr/}bin/cpio ixr,
  /{,usr/}bin/cut ixr,
  /{,usr/}bin/date ixr,
  /{,usr/}bin/dd ixr,
  /{,usr/}bin/diff{,3} ixr,
  /{,usr/}bin/dir ixr,
  /{,usr/}bin/dirname ixr,
  /{,usr/}bin/echo ixr,
  /{,usr/}bin/{,e,f,r}grep ixr,
  /{,usr/}bin/env ixr,
  /{,usr/}bin/expr ixr,
  /{,usr/}bin/find ixr,
  /{,usr/}bin/fmt ixr,
  /{,usr/}bin/getopt ixr,
  /{,usr/}bin/false ixr,
  /{,usr/}bin/head ixr,
  /{,usr/}bin/id ixr,
  /{,usr/}bin/igawk ixr,
  /{,usr/}bin/kill ixr,
  /{,usr/}bin/ln ixr,
  /{,usr/}bin/line ixr,
  /{,usr/}bin/link ixr,
  /{,usr/}bin/ls ixr,
  /{,usr/}bin/md5sum ixr,
  /{,usr/}bin/mkdir ixr,
  /{,usr/}bin/mktemp ixr,
  /{,usr/}bin/mv ixr,
  /{,usr/}bin/pgrep ixr,
  /{,usr/}bin/printenv ixr,
  /{,usr/}bin/printf ixr,
  /{,usr/}bin/ps ixr,
  /{,usr/}bin/pwd ixr,
  /{,usr/}bin/readlink ixr,
  /{,usr/}bin/realpath ixr,
  /{,usr/}bin/rev ixr,
  /{,usr/}bin/rm ixr,
  /{,usr/}bin/rmdir ixr,
  /{,usr/}bin/sed ixr,
  /{,usr/}bin/seq ixr,
  /{,usr/}bin/sleep ixr,
  /{,usr/}bin/sort ixr,
  /{,usr/}bin/stat ixr,
  /{,usr/}bin/tac ixr,
  /{,usr/}bin/tail ixr,
  /{,usr/}bin/tar ixr,
  /{,usr/}bin/tee ixr,
  /{,usr/}bin/test ixr,
  /{,usr/}bin/tempfile ixr,
  /{,usr/}bin/touch ixr,
  /{,usr/}bin/tr ixr,
  /{,usr/}bin/true ixr,
  /{,usr/}bin/uname ixr,
  /{,usr/}bin/uniq ixr,
  /{,usr/}bin/unlink ixr,
  /{,usr/}bin/unxz ixr,
  /{,usr/}bin/unzip ixr,
  /{,usr/}bin/vdir ixr,
  /{,usr/}bin/wc ixr,
  /{,usr/}bin/which ixr,
  /{,usr/}bin/xz ixr,
  /{,usr/}bin/yes ixr,
  /{,usr/}bin/zcat ixr,
  /{,usr/}bin/z{,e,f}grep ixr,
  /{,usr/}bin/zip ixr,
  /{,usr/}bin/zipgrep ixr,

  # uptime
  /{,usr/}bin/uptime ixr,
  @{PROC}/uptime r,
  @{PROC}/loadavg r,
  # this is an information leak
  deny /{,var/}run/utmp r,

  # Miscellaneous accesses
  /etc/mime.types r,
  @{PROC}/sys/kernel/hostname r,
  @{PROC}/sys/kernel/osrelease r,

  # Read-only for the install directory
  @{CLICK_DIR}/@{APP_PKGNAME}/                   r,
  @{CLICK_DIR}/@{APP_PKGNAME}/@{APP_VERSION}/    r,
  @{CLICK_DIR}/@{APP_PKGNAME}/@{APP_VERSION}/**  mrklix,

  # Read-only home area for other versions
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/                  r,
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/@{APP_VERSION}/   r,
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/@{APP_VERSION}/** mrkix,

  # Writable home area for this version.
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/@{APP_VERSION}/   w,
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/@{APP_VERSION}/** wl,

  # Read-only system area for other versions
  /var/lib/apps/@{APP_PKGNAME}/   r,
  /var/lib/apps/@{APP_PKGNAME}/** mrkix,

  # TODO: the write on these is needed in case they doesn't exist, but means an
  # app could adjust inode data and affect rollbacks.
  owner @{HOMEDIRS}/*/apps/@{APP_PKGNAME}/         w,
  /var/lib/apps/@{APP_PKGNAME}/                  w,

  # Writable system area only for this version
  /var/lib/apps/@{APP_PKGNAME}/@{APP_VERSION}/   w,
  /var/lib/apps/@{APP_PKGNAME}/@{APP_VERSION}/** wl,

  # Writable temp area only for this version (launcher will create this
  # directory on our behalf so only allow readonly on parent)
  /tmp/snapps/@{APP_PKGNAME}/                  r,
  /tmp/snapps/@{APP_PKGNAME}/**                rk,
  /tmp/snapps/@{APP_PKGNAME}/@{APP_VERSION}/   rw,
  /tmp/snapps/@{APP_PKGNAME}/@{APP_VERSION}/** mrwlkix,

  # No abstractions specified

  # Rules specified via policy groups
  # Description: Can access the network
  # Usage: common
  #include <abstractions/nameservice>
  #include <abstractions/ssl_certs>

  @{PROC}/sys/net/core/somaxconn r,

  # We want to explicitly deny access to NetworkManager because its DBus API
  # gives away too much
  deny dbus (receive, send)
       bus=system
       path=/org/freedesktop/NetworkManager,
  deny dbus (receive, send)
       bus=system
       peer=(name=org.freedesktop.NetworkManager),

  # Do the same for ofono (LP: #1226844)
  deny dbus (receive, send)
       bus=system
       interface="org.ofono.Manager",

  # Specified read permissions
  /etc/hosts.allow rk,
  /etc/hosts.deny rk,
  /etc/passwd rk,
  /proc/cmdline rk,
  /sys/bus/i2c/devices/0-0050/eeprom rk,
  /sys/devices/ocp/44e0b000.i2c/i2c-0/0-0050/eeprom rk,
  @{PROC}/ rk,
  @{PROC}/** rk,
  @{PROC}/[0-9]*/stat rk,

  # Specified write permissions
  /sys/bus/i2c/devices/i2c-0/new_device rwk,

  # Ninja
  /{,usr/}bin/xxd ixr,
  /sys/class/net/[a-z0-9]*/address rk,
  /sys/devices/*/*/net/[a-z0-9]*/address rk,
  /proc/cmdline rk,
  /bin/ip ixr,
  /dev/gestic rwk,
  /dev/ttyO3 rwk,
  /dev/tty.ledmatrix rwk,
  /sys/class/gpio rwk,
  /sys/class/gpio/** rwk,
  /sys/devices/virtual/gpio/** rwk,

  # Access back to the framework
  /apps/ninjasphere/*/sphere-schemas/** rk,
  /apps/ninjasphere/*/config/** rk,
  /apps/ninjasphere/*/bin/** ixr,
}`
<|endoftext|>


<commit_before>package resource

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Repository struct {
	dir     string
	keyFile string
	source  string
}

func NewRepository(source string, dir string, keyFile string) Repository {
	return Repository{
		dir:     dir,
		keyFile: keyFile,
		source:  source,
	}
}

func (r Repository) Clone() error {
	cmd := exec.Command("git", "clone", r.source, r.dir)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=\"ssh -i %s\"", r.keyFile))
	}
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Could not clone repository: git clone %s %s failed: %s\n[STDERR]\n%s", r.source, r.dir, err, errBytes.String())
	}
	return nil
}

func (r Repository) Fetch() error {
	err := r.runRepoCmd("git", "fetch", "origin")
	if err != nil {
		return fmt.Errorf("Could not fetch origin: %s", err)
	}
	return nil
}

func (r Repository) CheckoutRef(ref string) error {
	err := r.runRepoCmd("git", "checkout", ref)
	if err != nil {
		return fmt.Errorf("Could not checkout %s: %s", ref, err)
	}
	return nil
}

func (r Repository) RemoteBranches() ([]string, error) {
	branchesOutput, err := r.runRepoCmdOutput("git", "branch", "-r")
	if err != nil {
		return []string{}, fmt.Errorf("Could not list remote branches: %s", err)
	}
	branches := strings.Split(branchesOutput, "\n")

	trimmedBranches := []string{}
	for _, branch := range branches {
		if !strings.Contains(branch, "origin/HEAD ->") {
			trimmedBranches = append(trimmedBranches, strings.TrimSpace(branch))
		}
	}
	return trimmedBranches, nil
}

func (r Repository) RefAuthorName(ref string) (string, error) {
	nameOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%an\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show author name for %s: %s", ref, err)
	}
	return strings.Trim(nameOutput, "\""), nil
}

func (r Repository) RefAuthorDate(ref string) (string, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ai\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show author date for %s: %s", ref, err)
	}
	return strings.Trim(timeOutput, "\""), nil
}

func (r Repository) RefCommitName(ref string) (string, error) {
	nameOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%cn\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show committer name for %s: %s", ref, err)
	}
	return strings.Trim(nameOutput, "\""), nil
}

func (r Repository) RefCommitDate(ref string) (string, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ci\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show committer date for %s: %s", ref, err)
	}
	return strings.Trim(timeOutput, "\""), nil
}

func (r Repository) RefCommitTimestamp(ref string) (int64, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ct\"", ref)
	if err != nil {
		return 0, fmt.Errorf("Could not show committer timestamp for %s: %s", ref, err)
	}
	timeString := strings.Trim(timeOutput, "\"")
	timestamp, err := strconv.ParseInt(timeString, 10, 64)
	if err != nil {
		return 0, fmt.Errorf("Could not parse committer timestamp (%s) for %s: %s", timeString, ref, err)
	}
	return timestamp, nil
}

func (r Repository) RefMessage(ref string) (string, error) {
	msgOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%B\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show message for %s: %s", ref, err)
	}
	return strings.Trim(msgOutput, "\""), nil
}

func (r Repository) LatestRef(branch string) (string, error) {
	refOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%H\"", branch)
	if err != nil {
		return "", fmt.Errorf("Could not show SHA for %s: %s", branch, err)
	}
	return strings.Trim(refOutput, "\""), nil
}

func (r Repository) RefsSinceTimestamp(branch string, timestamp int64) ([]string, error) {
	refsOutput, err := r.runRepoCmdOutput("git", "log", fmt.Sprintf("--since=%d", timestamp), "--format=\"%H\"", branch)
	if err != nil {
		return []string{}, fmt.Errorf("Could not list refs since %d for %s: %s", timestamp, branch, err)
	}
	return strings.Split(refsOutput, "\n"), nil
}

func (r Repository) runRepoCmd(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=\"ssh -i %s\"", r.keyFile))
	}
	cmd.Dir = r.dir
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("%s %v in %s failed: %s\n[STDERR]\n%s", name, args, r.dir, err, errBytes.String())
	}
	return nil
}

func (r Repository) runRepoCmdOutput(name string, args ...string) (string, error) {
	cmd := exec.Command(name, args...)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=\"ssh -i %s\"", r.keyFile))
	}
	cmd.Dir = r.dir
	var outputBytes bytes.Buffer
	cmd.Stdout = &outputBytes
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("%s %v in %s failed: %s\n[STDERR]\n%s", name, args, r.dir, err, errBytes.String())
	}
	return strings.TrimSpace(outputBytes.String()), nil
}

func CreateKeyFile(privateKey string) (string, error) {
	keyFile, err := ioutil.TempFile("", "tracker-git-branch-resource")
	if err != nil {
		return "", fmt.Errorf("Could not create keyfile: %s", err)
	}
	keyFile.Chmod(0600)
	_, err = keyFile.WriteString(privateKey)
	if err != nil {
		return "", fmt.Errorf("Could not write keyfile %s: %s", keyFile, err)
	}
	return keyFile.Name(), nil
}
<commit_msg>Try a full path to ssh<commit_after>package resource

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Repository struct {
	dir     string
	keyFile string
	source  string
}

func NewRepository(source string, dir string, keyFile string) Repository {
	return Repository{
		dir:     dir,
		keyFile: keyFile,
		source:  source,
	}
}

func (r Repository) Clone() error {
	cmd := exec.Command("git", "clone", r.source, r.dir)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=\"/usr/bin/ssh -i %s\"", r.keyFile))
	}
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Could not clone repository: git clone %s %s failed: %s\n[STDERR]\n%s", r.source, r.dir, err, errBytes.String())
	}
	return nil
}

func (r Repository) Fetch() error {
	err := r.runRepoCmd("git", "fetch", "origin")
	if err != nil {
		return fmt.Errorf("Could not fetch origin: %s", err)
	}
	return nil
}

func (r Repository) CheckoutRef(ref string) error {
	err := r.runRepoCmd("git", "checkout", ref)
	if err != nil {
		return fmt.Errorf("Could not checkout %s: %s", ref, err)
	}
	return nil
}

func (r Repository) RemoteBranches() ([]string, error) {
	branchesOutput, err := r.runRepoCmdOutput("git", "branch", "-r")
	if err != nil {
		return []string{}, fmt.Errorf("Could not list remote branches: %s", err)
	}
	branches := strings.Split(branchesOutput, "\n")

	trimmedBranches := []string{}
	for _, branch := range branches {
		if !strings.Contains(branch, "origin/HEAD ->") {
			trimmedBranches = append(trimmedBranches, strings.TrimSpace(branch))
		}
	}
	return trimmedBranches, nil
}

func (r Repository) RefAuthorName(ref string) (string, error) {
	nameOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%an\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show author name for %s: %s", ref, err)
	}
	return strings.Trim(nameOutput, "\""), nil
}

func (r Repository) RefAuthorDate(ref string) (string, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ai\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show author date for %s: %s", ref, err)
	}
	return strings.Trim(timeOutput, "\""), nil
}

func (r Repository) RefCommitName(ref string) (string, error) {
	nameOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%cn\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show committer name for %s: %s", ref, err)
	}
	return strings.Trim(nameOutput, "\""), nil
}

func (r Repository) RefCommitDate(ref string) (string, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ci\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show committer date for %s: %s", ref, err)
	}
	return strings.Trim(timeOutput, "\""), nil
}

func (r Repository) RefCommitTimestamp(ref string) (int64, error) {
	timeOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%ct\"", ref)
	if err != nil {
		return 0, fmt.Errorf("Could not show committer timestamp for %s: %s", ref, err)
	}
	timeString := strings.Trim(timeOutput, "\"")
	timestamp, err := strconv.ParseInt(timeString, 10, 64)
	if err != nil {
		return 0, fmt.Errorf("Could not parse committer timestamp (%s) for %s: %s", timeString, ref, err)
	}
	return timestamp, nil
}

func (r Repository) RefMessage(ref string) (string, error) {
	msgOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%B\"", ref)
	if err != nil {
		return "", fmt.Errorf("Could not show message for %s: %s", ref, err)
	}
	return strings.Trim(msgOutput, "\""), nil
}

func (r Repository) LatestRef(branch string) (string, error) {
	refOutput, err := r.runRepoCmdOutput("git", "show", "-s", "--format=\"%H\"", branch)
	if err != nil {
		return "", fmt.Errorf("Could not show SHA for %s: %s", branch, err)
	}
	return strings.Trim(refOutput, "\""), nil
}

func (r Repository) RefsSinceTimestamp(branch string, timestamp int64) ([]string, error) {
	refsOutput, err := r.runRepoCmdOutput("git", "log", fmt.Sprintf("--since=%d", timestamp), "--format=\"%H\"", branch)
	if err != nil {
		return []string{}, fmt.Errorf("Could not list refs since %d for %s: %s", timestamp, branch, err)
	}
	return strings.Split(refsOutput, "\n"), nil
}

func (r Repository) runRepoCmd(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=\"/usr/bin/ssh -i %s\"", r.keyFile))
	}
	cmd.Dir = r.dir
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("%s %v in %s failed: %s\n[STDERR]\n%s", name, args, r.dir, err, errBytes.String())
	}
	return nil
}

func (r Repository) runRepoCmdOutput(name string, args ...string) (string, error) {
	cmd := exec.Command(name, args...)
	if r.keyFile != "" {
		cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=\"/usr/bin/ssh -i %s\"", r.keyFile))
	}
	cmd.Dir = r.dir
	var outputBytes bytes.Buffer
	cmd.Stdout = &outputBytes
	var errBytes bytes.Buffer
	cmd.Stderr = &errBytes
	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("%s %v in %s failed: %s\n[STDERR]\n%s", name, args, r.dir, err, errBytes.String())
	}
	return strings.TrimSpace(outputBytes.String()), nil
}

func CreateKeyFile(privateKey string) (string, error) {
	keyFile, err := ioutil.TempFile("", "tracker-git-branch-resource")
	if err != nil {
		return "", fmt.Errorf("Could not create keyfile: %s", err)
	}
	keyFile.Chmod(0600)
	_, err = keyFile.WriteString(privateKey)
	if err != nil {
		return "", fmt.Errorf("Could not write keyfile %s: %s", keyFile, err)
	}
	return keyFile.Name(), nil
}
<|endoftext|>


<commit_before>package models

import "net/http"

//@Title  返回数据结构处理
//@Author cbping
type RespInfo struct {
	F_responseNo  int         `required:"true" description:"响应码"`
	F_responseMsg string      `description:"响应码描述"`
	F_data        interface{} `description:"响应内容"`
}

//响应码
//@Description
const (
	//commom
	RESP_OK = 10000
	RESP_ERR = 10001
	RESP_PARAM_ERR = 10002
	RESP_TOKEN_ERR = 10003
	RESP_NO_ACCESS = 10004
	RESP_APP_NOT_ON = 10005
	RESP_UNKNOWN_ERR = 10006
)

//响应码描述
var respMsg map[int]string = map[int]string{
	RESP_OK:        "操作成功",
	RESP_ERR:       "操作失败",
	RESP_PARAM_ERR: "参数错误",
	RESP_TOKEN_ERR: "签名认证错误",
	RESP_NO_ACCESS: "对不起，您没有此操作权限",
	RESP_APP_NOT_ON: "暂时未提供服务",
	RESP_UNKNOWN_ERR: "未知错误",

	http.StatusBadRequest:        "参数有误（Bad Request）",
	http.StatusUnauthorized:        "用户认证不成功",
	http.StatusForbidden:           "拒绝访问",
	http.StatusNotFound:            "资源不存在",
	http.StatusMethodNotAllowed:    "此方法未允许访问",
	http.StatusRequestTimeout:    "请求远程服务器超时",
	http.StatusInternalServerError: "服务器内部错误",
	http.StatusServiceUnavailable:  "服务不可用",
}

//
func NewRespInfo(code int, msg string, data interface{}) (respInfo *RespInfo) {
	if "" == msg {
		msg = GetMsgWithCode(code)
	}
	respInfo = &RespInfo{code, msg, data}
	return
}

//
func NewRespInfoWithCode(code int, data interface{}) (respInfo *RespInfo) {
	respInfo = NewRespInfo(code, "", data)
	return
}

func GetMsgWithCode(code int) string {
	val, ok := respMsg[code]
	if !ok {
		val = ""
	}
	return val
}

// @Title 转化成map
// @Description
// @Return map[string]interface{}
func (ri *RespInfo) ToStringMap() (res map[string]interface{}) {
	res = make(map[string]interface{})
	res["F_responseNo"] = ri.F_responseNo
	res["F_responseMsg"] = ri.F_responseMsg
	if nil != ri.F_data {
		res["F_data"] = ri.F_data
	}
	return
}

// @Title SetData
// @Description
func (ri *RespInfo) SetData(data interface{}) {
	ri.F_data = data
}
<commit_msg>go fmt<commit_after>package models

import "net/http"

//@Title  返回数据结构处理
//@Author cbping
type RespInfo struct {
	F_responseNo  int         `required:"true" description:"响应码"`
	F_responseMsg string      `description:"响应码描述"`
	F_data        interface{} `description:"响应内容"`
}

//响应码
//@Description
const (
	//commom
	RESP_OK          = 10000
	RESP_ERR         = 10001
	RESP_PARAM_ERR   = 10002
	RESP_TOKEN_ERR   = 10003
	RESP_NO_ACCESS   = 10004
	RESP_APP_NOT_ON  = 10005
	RESP_UNKNOWN_ERR = 10006
)

//响应码描述
var respMsg map[int]string = map[int]string{
	RESP_OK:          "操作成功",
	RESP_ERR:         "操作失败",
	RESP_PARAM_ERR:   "参数错误",
	RESP_TOKEN_ERR:   "签名认证错误",
	RESP_NO_ACCESS:   "对不起，您没有此操作权限",
	RESP_APP_NOT_ON:  "暂时未提供服务",
	RESP_UNKNOWN_ERR: "未知错误",

	http.StatusBadRequest:          "参数有误（Bad Request）",
	http.StatusUnauthorized:        "用户认证不成功",
	http.StatusForbidden:           "拒绝访问",
	http.StatusNotFound:            "资源不存在",
	http.StatusMethodNotAllowed:    "此方法未允许访问",
	http.StatusRequestTimeout:      "请求远程服务器超时",
	http.StatusInternalServerError: "服务器内部错误",
	http.StatusServiceUnavailable:  "服务不可用",
}

//
func NewRespInfo(code int, msg string, data interface{}) (respInfo *RespInfo) {
	if "" == msg {
		msg = GetMsgWithCode(code)
	}
	respInfo = &RespInfo{code, msg, data}
	return
}

//
func NewRespInfoWithCode(code int, data interface{}) (respInfo *RespInfo) {
	respInfo = NewRespInfo(code, "", data)
	return
}

func GetMsgWithCode(code int) string {
	val, ok := respMsg[code]
	if !ok {
		val = ""
	}
	return val
}

// @Title 转化成map
// @Description
// @Return map[string]interface{}
func (ri *RespInfo) ToStringMap() (res map[string]interface{}) {
	res = make(map[string]interface{})
	res["F_responseNo"] = ri.F_responseNo
	res["F_responseMsg"] = ri.F_responseMsg
	if nil != ri.F_data {
		res["F_data"] = ri.F_data
	}
	return
}

// @Title SetData
// @Description
func (ri *RespInfo) SetData(data interface{}) {
	ri.F_data = data
}
<|endoftext|>


<commit_before>package renter

import (
	"errors"
	"io"
	"net"
	"os"
	"path/filepath"
	"sync/atomic"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

type uploadPiece struct {
	data       []byte
	chunkIndex uint64
	pieceIndex uint64
}

// An uploader uploads pieces to a host. This interface exists to facilitate
// easy testing.
type uploader interface {
	// addPiece uploads a piece to the uploader.
	addPiece(uploadPiece) (*fileContract, error)
}

// A hostUploader uploads pieces to a host. It implements the uploader interface.
type hostUploader struct {
	conn      net.Conn
	masterKey crypto.TwofishKey
	contract  fileContract
	renter    *Renter
}

// addPiece uploads a piece to a host, and returns the updated fileContract.
func (hu *hostUploader) addPiece(p uploadPiece) (*fileContract, error) {
	// encrypt piece data
	key := deriveKey(hu.masterKey, p.chunkIndex, p.pieceIndex)
	encPiece, err := key.EncryptBytes(p.data)
	if err != nil {
		return nil, err
	}

	// revise contract with host (see negotiate.go)
	offset, err := hu.renter.revise(hu.contract.ID, encPiece)
	if err != nil {
		return nil, err
	}

	// update fileContract
	hu.contract.Pieces = append(hu.contract.Pieces, pieceData{
		Chunk:  p.chunkIndex,
		Piece:  p.pieceIndex,
		Offset: offset,
		Length: uint64(len(encPiece)),
	})

	return &hu.contract, nil
}

// newHostUploader establishes a connection to a host.
func (r *Renter) newHostUploader(host modules.HostSettings, masterKey crypto.TwofishKey) (*hostUploader, error) {
	conn, err := net.DialTimeout("tcp", string(host.IPAddress), 5*time.Second)
	if err != nil {
		return nil, err
	}

	// initialize file contract (see negotiate.go)
	fcid, err := r.negotiateContract(conn, host)
	if err != nil {
		return nil, err
	}

	return &hostUploader{
		conn:      conn,
		masterKey: masterKey,
		contract:  fileContract{ID: fcid, IP: host.IPAddress},
		renter:    r,
	}, nil
}

// uploadWorker uploads pieces to a host as directed by reqChan. When there
// are no more pieces to upload, it sends the final version of the
// fileContract down respChan.
func uploadWorker(host uploader, reqChan chan uploadPiece, respChan chan *fileContract) {
	var contract *fileContract
	var err error
	for req := range reqChan {
		contract, err = host.addPiece(req)
		if err != nil {
			// TODO: how should this be handled?
			break
		}
	}
	// reqChan has been closed; send final contract
	respChan <- contract
}

// upload reads chunks from r and uploads them to hosts. It spawns a worker
// for each host, and instructs them to upload pieces of each chunk.
func (f *file) upload(r io.Reader, hosts []uploader) error {
	// All requests are sent down the same channel. Since all workers are
	// waiting on this channel, pieces will be uploaded by the first idle
	// worker. This means faster uploaders will get more pieces than slow
	// uploaders.
	reqChan := make(chan uploadPiece)

	// Once all requests have been sent, upload will read the resulting
	// fileContracts from respChan and store them in f.
	respChan := make(chan *fileContract)

	// spawn workers
	for _, h := range hosts {
		go uploadWorker(h, reqChan, respChan)
	}

	// encode and upload each chunk
	for i := uint64(0); ; i++ {
		// read next chunk
		chunk := make([]byte, f.chunkSize())
		_, err := io.ReadFull(r, chunk)
		if err == io.EOF {
			break
		} else if err != nil && err != io.ErrUnexpectedEOF {
			return err
		}
		// encode
		pieces, err := f.ecc.Encode(chunk)
		if err != nil {
			return err
		}
		// send upload requests to workers
		uploaded := 0
		for j, data := range pieces {
			reqChan <- uploadPiece{data, i, uint64(j)}
			uploaded += len(data)
		}
		atomic.AddUint64(&f.bytesUploaded, uint64(uploaded)) // TODO: move inside workers
		atomic.AddUint64(&f.chunksUploaded, 1)
	}

	// signal workers to send their contracts
	close(reqChan)
	for range hosts {
		contract := <-respChan
		if contract == nil {
			continue
		}
		f.Contracts[contract.IP] = *contract
	}

	return nil
}

// checkWalletBalance looks at an upload and determines if there is enough
// money in the wallet to support such an upload. An error is returned if it is
// determined that there is not enough money.
func (r *Renter) checkWalletBalance(up modules.FileUploadParams) error {
	// Get the size of the file.
	fileInfo, err := os.Stat(up.Filename)
	if err != nil {
		return err
	}
	curSize := types.NewCurrency64(uint64(fileInfo.Size()))

	var averagePrice types.Currency
	sampleSize := up.ECC.NumPieces() * 3 / 2
	hosts := r.hostDB.RandomHosts(sampleSize)
	for _, host := range hosts {
		averagePrice = averagePrice.Add(host.Price)
	}
	if len(hosts) == 0 {
		return errors.New("no hosts!")
	}
	averagePrice = averagePrice.Div(types.NewCurrency64(uint64(len(hosts))))
	estimatedCost := averagePrice.Mul(types.NewCurrency64(uint64(up.Duration))).Mul(curSize)
	bufferedCost := estimatedCost.Mul(types.NewCurrency64(2))

	siacoinBalance, _, _ := r.wallet.ConfirmedBalance()
	if bufferedCost.Cmp(siacoinBalance) > 0 {
		return errors.New("insufficient balance for upload")
	}
	return nil
}

// Upload takes an upload parameters, which contain a file to upload, and then
// creates a redundant copy of the file on the Sia network.
func (r *Renter) Upload(up modules.FileUploadParams) error {
	// TODO: This type of restriction is something that should be handled by
	// the frontend, not the backend.
	if filepath.Ext(up.Filename) != filepath.Ext(up.Nickname) {
		return errors.New("nickname and file name must have the same extension")
	}

	// Open the file.
	handle, err := os.Open(up.Filename)
	if err != nil {
		return err
	}

	// Check that we have enough money to finance the upload.
	err = r.checkWalletBalance(up)
	if err != nil {
		return err
	}

	// Check for a nickname conflict.
	lockID := r.mu.RLock()
	_, exists := r.files[up.Nickname]
	r.mu.RUnlock(lockID)
	if exists {
		return errors.New("file with that nickname already exists")
	}

	// Check that the file is less than 5 GiB.
	fileInfo, err := handle.Stat()
	if err != nil {
		return err
	}
	// NOTE: The upload max of 5 GiB is temporary and therefore does not have
	// a constant. This should be removed once micropayments + upload resuming
	// are in place. 5 GiB is chosen to prevent confusion - on anybody's
	// machine any file appearing to be under 5 GB will be below the hard
	// limit.
	if fileInfo.Size() > 5*1024*1024*1024 {
		return errors.New("cannot upload a file larger than 5 GB")
	}

	// Check that the hostdb is sufficiently large to support an upload.
	// TODO: ActiveHosts needs to only report hosts >= v0.4
	if len(r.hostDB.ActiveHosts()) < up.ECC.NumPieces() {
		return errors.New("not enough hosts on the network to upload a file")
	}

	// Create file object.
	f := newFile(up.ECC, up.PieceSize, uint64(fileInfo.Size()))

	// Add file to renter.
	lockID = r.mu.Lock()
	r.files[up.Nickname] = f
	r.save()
	r.mu.Unlock(lockID)

	// Upload to hosts in parallel.
	var hosts []uploader
	for _, host := range r.hostDB.RandomHosts(up.ECC.NumPieces()) {
		host, err := r.newHostUploader(host, f.MasterKey)
		if err != nil {
			return err
		}
		defer host.conn.Close()
		hosts = append(hosts, host)
	}
	err = f.upload(handle, hosts)
	if err != nil {
		// Upload failed; remove the file object.
		lockID = r.mu.Lock()
		delete(r.files, up.Nickname)
		r.save()
		r.mu.Unlock(lockID)
		return errors.New("failed to upload any file pieces")
	}

	// Save the .sia file to the renter directory.
	err = r.saveFile(f)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>move UploadProgress update to workers<commit_after>package renter

import (
	"errors"
	"io"
	"net"
	"os"
	"path/filepath"
	"sync/atomic"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

type uploadPiece struct {
	data       []byte
	chunkIndex uint64
	pieceIndex uint64
}

// An uploader uploads pieces to a host. This interface exists to facilitate
// easy testing.
type uploader interface {
	// addPiece uploads a piece to the uploader.
	addPiece(uploadPiece) (*fileContract, error)
}

// A hostUploader uploads pieces to a host. It implements the uploader interface.
type hostUploader struct {
	conn      net.Conn
	masterKey crypto.TwofishKey
	contract  fileContract
	renter    *Renter
}

// addPiece uploads a piece to a host, and returns the updated fileContract.
func (hu *hostUploader) addPiece(p uploadPiece) (*fileContract, error) {
	// encrypt piece data
	key := deriveKey(hu.masterKey, p.chunkIndex, p.pieceIndex)
	encPiece, err := key.EncryptBytes(p.data)
	if err != nil {
		return nil, err
	}

	// revise contract with host (see negotiate.go)
	offset, err := hu.renter.revise(hu.contract.ID, encPiece)
	if err != nil {
		return nil, err
	}

	// update fileContract
	hu.contract.Pieces = append(hu.contract.Pieces, pieceData{
		Chunk:  p.chunkIndex,
		Piece:  p.pieceIndex,
		Offset: offset,
		Length: uint64(len(encPiece)),
	})

	return &hu.contract, nil
}

// newHostUploader establishes a connection to a host.
func (r *Renter) newHostUploader(host modules.HostSettings, masterKey crypto.TwofishKey) (*hostUploader, error) {
	conn, err := net.DialTimeout("tcp", string(host.IPAddress), 5*time.Second)
	if err != nil {
		return nil, err
	}

	// initialize file contract (see negotiate.go)
	fcid, err := r.negotiateContract(conn, host)
	if err != nil {
		return nil, err
	}

	return &hostUploader{
		conn:      conn,
		masterKey: masterKey,
		contract:  fileContract{ID: fcid, IP: host.IPAddress},
		renter:    r,
	}, nil
}

// uploadWorker uploads pieces to a host as directed by reqChan. When there
// are no more pieces to upload, it sends the final version of the
// fileContract down respChan.
func (f *file) uploadWorker(host uploader, reqChan chan uploadPiece, respChan chan *fileContract) {
	var contract *fileContract
	var err error
	for req := range reqChan {
		contract, err = host.addPiece(req)
		if err != nil {
			// TODO: how should this be handled?
			break
		}
		atomic.AddUint64(&f.bytesUploaded, uint64(len(req.data)))

	}
	// reqChan has been closed; send final contract
	respChan <- contract
}

// upload reads chunks from r and uploads them to hosts. It spawns a worker
// for each host, and instructs them to upload pieces of each chunk.
func (f *file) upload(r io.Reader, hosts []uploader) error {
	// All requests are sent down the same channel. Since all workers are
	// waiting on this channel, pieces will be uploaded by the first idle
	// worker. This means faster uploaders will get more pieces than slow
	// uploaders.
	reqChan := make(chan uploadPiece)

	// Once all requests have been sent, upload will read the resulting
	// fileContracts from respChan and store them in f.
	respChan := make(chan *fileContract)

	// spawn workers
	for _, h := range hosts {
		go f.uploadWorker(h, reqChan, respChan)
	}

	// encode and upload each chunk
	for i := uint64(0); ; i++ {
		// read next chunk
		chunk := make([]byte, f.chunkSize())
		_, err := io.ReadFull(r, chunk)
		if err == io.EOF {
			break
		} else if err != nil && err != io.ErrUnexpectedEOF {
			return err
		}
		// encode
		pieces, err := f.ecc.Encode(chunk)
		if err != nil {
			return err
		}
		// send upload requests to workers
		for j, data := range pieces {
			reqChan <- uploadPiece{data, i, uint64(j)}
		}
		atomic.AddUint64(&f.chunksUploaded, 1)
	}

	// signal workers to send their contracts
	close(reqChan)
	for range hosts {
		contract := <-respChan
		if contract == nil {
			continue
		}
		f.Contracts[contract.IP] = *contract
	}

	return nil
}

// checkWalletBalance looks at an upload and determines if there is enough
// money in the wallet to support such an upload. An error is returned if it is
// determined that there is not enough money.
func (r *Renter) checkWalletBalance(up modules.FileUploadParams) error {
	// Get the size of the file.
	fileInfo, err := os.Stat(up.Filename)
	if err != nil {
		return err
	}
	curSize := types.NewCurrency64(uint64(fileInfo.Size()))

	var averagePrice types.Currency
	sampleSize := up.ECC.NumPieces() * 3 / 2
	hosts := r.hostDB.RandomHosts(sampleSize)
	for _, host := range hosts {
		averagePrice = averagePrice.Add(host.Price)
	}
	if len(hosts) == 0 {
		return errors.New("no hosts!")
	}
	averagePrice = averagePrice.Div(types.NewCurrency64(uint64(len(hosts))))
	estimatedCost := averagePrice.Mul(types.NewCurrency64(uint64(up.Duration))).Mul(curSize)
	bufferedCost := estimatedCost.Mul(types.NewCurrency64(2))

	siacoinBalance, _, _ := r.wallet.ConfirmedBalance()
	if bufferedCost.Cmp(siacoinBalance) > 0 {
		return errors.New("insufficient balance for upload")
	}
	return nil
}

// Upload takes an upload parameters, which contain a file to upload, and then
// creates a redundant copy of the file on the Sia network.
func (r *Renter) Upload(up modules.FileUploadParams) error {
	// TODO: This type of restriction is something that should be handled by
	// the frontend, not the backend.
	if filepath.Ext(up.Filename) != filepath.Ext(up.Nickname) {
		return errors.New("nickname and file name must have the same extension")
	}

	// Open the file.
	handle, err := os.Open(up.Filename)
	if err != nil {
		return err
	}

	// Check that we have enough money to finance the upload.
	err = r.checkWalletBalance(up)
	if err != nil {
		return err
	}

	// Check for a nickname conflict.
	lockID := r.mu.RLock()
	_, exists := r.files[up.Nickname]
	r.mu.RUnlock(lockID)
	if exists {
		return errors.New("file with that nickname already exists")
	}

	// Check that the file is less than 5 GiB.
	fileInfo, err := handle.Stat()
	if err != nil {
		return err
	}
	// NOTE: The upload max of 5 GiB is temporary and therefore does not have
	// a constant. This should be removed once micropayments + upload resuming
	// are in place. 5 GiB is chosen to prevent confusion - on anybody's
	// machine any file appearing to be under 5 GB will be below the hard
	// limit.
	if fileInfo.Size() > 5*1024*1024*1024 {
		return errors.New("cannot upload a file larger than 5 GB")
	}

	// Check that the hostdb is sufficiently large to support an upload.
	// TODO: ActiveHosts needs to only report hosts >= v0.4
	if len(r.hostDB.ActiveHosts()) < up.ECC.NumPieces() {
		return errors.New("not enough hosts on the network to upload a file")
	}

	// Create file object.
	f := newFile(up.ECC, up.PieceSize, uint64(fileInfo.Size()))

	// Add file to renter.
	lockID = r.mu.Lock()
	r.files[up.Nickname] = f
	r.save()
	r.mu.Unlock(lockID)

	// Upload to hosts in parallel.
	var hosts []uploader
	for _, host := range r.hostDB.RandomHosts(up.ECC.NumPieces()) {
		host, err := r.newHostUploader(host, f.MasterKey)
		if err != nil {
			return err
		}
		defer host.conn.Close()
		hosts = append(hosts, host)
	}
	err = f.upload(handle, hosts)
	if err != nil {
		// Upload failed; remove the file object.
		lockID = r.mu.Lock()
		delete(r.files, up.Nickname)
		r.save()
		r.mu.Unlock(lockID)
		return errors.New("failed to upload any file pieces")
	}

	// Save the .sia file to the renter directory.
	err = r.saveFile(f)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>