<commit_before>package lfs

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strconv"
	"strings"

	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
)

// An entry from ls-tree or rev-list including a blob sha and tree path
type TreeBlob struct {
	Sha1     string
	Filename string
}

func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.Filter) error {
	// We don't use the nameMap approach here since that's imprecise when >1 file
	// can be using the same content
	treeShas, err := lsTreeBlobs(ref, filter)
	if err != nil {
		return err
	}

	pcw, err := catFileBatchTree(treeShas)
	if err != nil {
		return err
	}

	for p := range pcw.Results {
		cb(p, nil)
	}

	if err := pcw.Wait(); err != nil {
		cb(nil, err)
	}
	return nil
}

// catFileBatchTree uses git cat-file --batch to get the object contents
// of a git object, given its sha1. The contents will be decoded into
// a Git LFS pointer. treeblobs is a channel over which blob entries
// will be sent. It returns a channel from which point.Pointers can be read.
func catFileBatchTree(treeblobs *TreeBlobChannelWrapper) (*PointerChannelWrapper, error) {
	scanner, err := NewPointerScanner()
	if err != nil {
		return nil, err
	}

	pointers := make(chan *WrappedPointer, chanBufSize)
	errchan := make(chan error, 10) // Multiple errors possible

	go func() {
		for t := range treeblobs.Results {
			hasNext := scanner.Scan(t.Sha1)
			if p := scanner.Pointer(); p != nil {
				p.Name = t.Filename
				pointers <- p
			}

			if err := scanner.Err(); err != nil {
				errchan <- err
			}

			if !hasNext {
				break
			}
		}

		// Deal with nested error from incoming treeblobs
		err := treeblobs.Wait()
		if err != nil {
			errchan <- err
		}

		if err = scanner.Close(); err != nil {
			errchan <- err
		}

		close(pointers)
		close(errchan)
	}()

	return NewPointerChannelWrapper(pointers, errchan), nil
}

// Use ls-tree at ref to find a list of candidate tree blobs which might be lfs files
// The returned channel will be sent these blobs which should be sent to catFileBatchTree
// for final check & conversion to Pointer
func lsTreeBlobs(ref string, filter *filepathfilter.Filter) (*TreeBlobChannelWrapper, error) {
	cmd, err := git.LsTree(ref)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	blobs := make(chan TreeBlob, chanBufSize)
	errchan := make(chan error, 1)

	go func() {
		scanner := newLsTreeScanner(cmd.Stdout)
		for scanner.Scan() {
			if t := scanner.TreeBlob(); t != nil && filter.Allows(t.Filename) {
				blobs <- *t
			}
		}

		stderr, _ := ioutil.ReadAll(cmd.Stderr)
		err := cmd.Wait()
		if err != nil {
			errchan <- fmt.Errorf("Error in git ls-tree: %v %v", err, string(stderr))
		}
		close(blobs)
		close(errchan)
	}()

	return NewTreeBlobChannelWrapper(blobs, errchan), nil
}

type lsTreeScanner struct {
	s    *bufio.Scanner
	tree *TreeBlob
}

func newLsTreeScanner(r io.Reader) *lsTreeScanner {
	s := bufio.NewScanner(r)
	s.Split(scanNullLines)
	return &lsTreeScanner{s: s}
}

func (s *lsTreeScanner) TreeBlob() *TreeBlob {
	return s.tree
}

func (s *lsTreeScanner) Err() error {
	return nil
}

func (s *lsTreeScanner) Scan() bool {
	t, hasNext := s.next()
	s.tree = t
	return hasNext
}

func (s *lsTreeScanner) next() (*TreeBlob, bool) {
	hasNext := s.s.Scan()
	line := s.s.Text()
	parts := strings.SplitN(line, "\t", 2)
	if len(parts) < 2 {
		return nil, hasNext
	}

	attrs := strings.SplitN(parts[0], " ", 4)
	if len(attrs) < 4 {
		return nil, hasNext
	}

	if attrs[1] != "blob" {
		return nil, hasNext
	}

	sz, err := strconv.ParseInt(strings.TrimSpace(attrs[3]), 10, 64)
	if err != nil {
		return nil, hasNext
	}

	if sz < blobSizeCutoff {
		sha1 := attrs[2]
		filename := parts[1]
		return &TreeBlob{Sha1: sha1, Filename: filename}, hasNext
	}
	return nil, hasNext
}

func scanNullLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	if i := bytes.IndexByte(data, '\000'); i >= 0 {
		// We have a full null-terminated line.
		return i + 1, data[0:i], nil
	}

	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), data, nil
	}

	// Request more data.
	return 0, nil, nil
}
<commit_msg>lfs: avoid deadlock when object scanner returns early<commit_after>package lfs

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strconv"
	"strings"

	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
)

// An entry from ls-tree or rev-list including a blob sha and tree path
type TreeBlob struct {
	Sha1     string
	Filename string
}

func runScanTree(cb GitScannerFoundPointer, ref string, filter *filepathfilter.Filter) error {
	// We don't use the nameMap approach here since that's imprecise when >1 file
	// can be using the same content
	treeShas, err := lsTreeBlobs(ref, filter)
	if err != nil {
		return err
	}

	pcw, err := catFileBatchTree(treeShas)
	if err != nil {
		return err
	}

	for p := range pcw.Results {
		cb(p, nil)
	}

	if err := pcw.Wait(); err != nil {
		cb(nil, err)
	}
	return nil
}

// catFileBatchTree uses git cat-file --batch to get the object contents
// of a git object, given its sha1. The contents will be decoded into
// a Git LFS pointer. treeblobs is a channel over which blob entries
// will be sent. It returns a channel from which point.Pointers can be read.
func catFileBatchTree(treeblobs *TreeBlobChannelWrapper) (*PointerChannelWrapper, error) {
	scanner, err := NewPointerScanner()
	if err != nil {
		return nil, err
	}

	pointers := make(chan *WrappedPointer, chanBufSize)
	errchan := make(chan error, 10) // Multiple errors possible

	go func() {
		hasNext := true
		for t := range treeblobs.Results {
			hasNext = scanner.Scan(t.Sha1)

			if p := scanner.Pointer(); p != nil {
				p.Name = t.Filename
				pointers <- p
			}

			if err := scanner.Err(); err != nil {
				errchan <- err
			}

			if !hasNext {
				break
			}
		}

		// If the scanner quit early, we may still have treeblobs to
		// read, so waiting for it to close will cause a deadlock.
		if hasNext {
			// Deal with nested error from incoming treeblobs
			err := treeblobs.Wait()
			if err != nil {
				errchan <- err
			}
		}

		if err = scanner.Close(); err != nil {
			errchan <- err
		}

		close(pointers)
		close(errchan)
	}()

	return NewPointerChannelWrapper(pointers, errchan), nil
}

// Use ls-tree at ref to find a list of candidate tree blobs which might be lfs files
// The returned channel will be sent these blobs which should be sent to catFileBatchTree
// for final check & conversion to Pointer
func lsTreeBlobs(ref string, filter *filepathfilter.Filter) (*TreeBlobChannelWrapper, error) {
	cmd, err := git.LsTree(ref)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	blobs := make(chan TreeBlob, chanBufSize)
	errchan := make(chan error, 1)

	go func() {
		scanner := newLsTreeScanner(cmd.Stdout)
		for scanner.Scan() {
			if t := scanner.TreeBlob(); t != nil && filter.Allows(t.Filename) {
				blobs <- *t
			}
		}

		stderr, _ := ioutil.ReadAll(cmd.Stderr)
		err := cmd.Wait()
		if err != nil {
			errchan <- fmt.Errorf("Error in git ls-tree: %v %v", err, string(stderr))
		}
		close(blobs)
		close(errchan)
	}()

	return NewTreeBlobChannelWrapper(blobs, errchan), nil
}

type lsTreeScanner struct {
	s    *bufio.Scanner
	tree *TreeBlob
}

func newLsTreeScanner(r io.Reader) *lsTreeScanner {
	s := bufio.NewScanner(r)
	s.Split(scanNullLines)
	return &lsTreeScanner{s: s}
}

func (s *lsTreeScanner) TreeBlob() *TreeBlob {
	return s.tree
}

func (s *lsTreeScanner) Err() error {
	return nil
}

func (s *lsTreeScanner) Scan() bool {
	t, hasNext := s.next()
	s.tree = t
	return hasNext
}

func (s *lsTreeScanner) next() (*TreeBlob, bool) {
	hasNext := s.s.Scan()
	line := s.s.Text()
	parts := strings.SplitN(line, "\t", 2)
	if len(parts) < 2 {
		return nil, hasNext
	}

	attrs := strings.SplitN(parts[0], " ", 4)
	if len(attrs) < 4 {
		return nil, hasNext
	}

	if attrs[1] != "blob" {
		return nil, hasNext
	}

	sz, err := strconv.ParseInt(strings.TrimSpace(attrs[3]), 10, 64)
	if err != nil {
		return nil, hasNext
	}

	if sz < blobSizeCutoff {
		sha1 := attrs[2]
		filename := parts[1]
		return &TreeBlob{Sha1: sha1, Filename: filename}, hasNext
	}
	return nil, hasNext
}

func scanNullLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	if i := bytes.IndexByte(data, '\000'); i >= 0 {
		// We have a full null-terminated line.
		return i + 1, data[0:i], nil
	}

	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), data, nil
	}

	// Request more data.
	return 0, nil, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dao

import (
	"fmt"
	"time"

	"github.com/astaxie/beego/orm"
	"github.com/goharbor/harbor/src/common/models"
)

var orderMap = map[string]string {
	"name": "name asc",
	"+name": "name asc",
	"-name": "name desc",
	"creation_time": "creation_time asc",
	"+creation_time": "creation_time asc",
	"-creation_time": "creation_time desc",
	"update_time": "update_time asc",
	"+update_time": "update_time asc",
	"-update_time": "update_time desc",
}

// AddRepository adds a repo to the database.
func AddRepository(repo models.RepoRecord) error {
	if repo.ProjectID == 0 {
		return fmt.Errorf("invalid project ID: %d", repo.ProjectID)
	}

	o := GetOrmer()
	now := time.Now()
	repo.CreationTime = now
	repo.UpdateTime = now
	_, err := o.Insert(&repo)
	return err
}

// GetRepositoryByName ...
func GetRepositoryByName(name string) (*models.RepoRecord, error) {
	o := GetOrmer()
	r := models.RepoRecord{Name: name}
	err := o.Read(&r, "Name")
	if err == orm.ErrNoRows {
		return nil, nil
	}
	return &r, err
}

// DeleteRepository ...
func DeleteRepository(name string) error {
	o := GetOrmer()
	_, err := o.QueryTable("repository").Filter("name", name).Delete()
	return err
}

// UpdateRepository ...
func UpdateRepository(repo models.RepoRecord) error {
	o := GetOrmer()
	repo.UpdateTime = time.Now()
	_, err := o.Update(&repo)
	return err
}

// IncreasePullCount ...
func IncreasePullCount(name string) (err error) {
	o := GetOrmer()
	num, err := o.QueryTable("repository").Filter("name", name).Update(
		orm.Params{
			"pull_count":  orm.ColValue(orm.ColAdd, 1),
			"update_time": time.Now(),
		})
	if err != nil {
		return err
	}
	if num == 0 {
		return fmt.Errorf("Failed to increase repository pull count with name: %s", name)
	}
	return nil
}

//RepositoryExists returns whether the repository exists according to its name.
func RepositoryExists(name string) bool {
	o := GetOrmer()
	return o.QueryTable("repository").Filter("name", name).Exist()
}

//GetTopRepos returns the most popular repositories whose project ID is
// in projectIDs
func GetTopRepos(projectIDs []int64, n int) ([]*models.RepoRecord, error) {
	repositories := []*models.RepoRecord{}
	if len(projectIDs) == 0 {
		return repositories, nil
	}

	_, err := GetOrmer().QueryTable(&models.RepoRecord{}).
		Filter("project_id__in", projectIDs).
		OrderBy("-pull_count").
		Limit(n).
		All(&repositories)

	return repositories, err
}

// GetTotalOfRepositories ...
func GetTotalOfRepositories(query ...*models.RepositoryQuery) (int64, error) {
	sql, params := repositoryQueryConditions(query...)
	sql = `select count(*) ` + sql
	var total int64
	if err := GetOrmer().Raw(sql, params).QueryRow(&total); err != nil {
		return 0, err
	}
	return total, nil
}

// GetRepositories ...
func GetRepositories(query ...*models.RepositoryQuery) ([]*models.RepoRecord, error) {
	repositories := []*models.RepoRecord{}
	order := "name asc"
	if len(query) > 0 && query[0] != nil {
		if s, ok := orderMap[query[0].Sort]; ok {
			order = s
		}
	}

	condition, params := repositoryQueryConditions(query...)
	sql := fmt.Sprintf(`select r.repository_id, r.name, r.project_id, r.description, r.pull_count,
	r.star_count, r.creation_time, r.update_time %s order by r.%s `, condition, order)
	if len(query) > 0 && query[0] != nil {
		page, size := query[0].Page, query[0].Size
		if size > 0 {
			sql += `limit ? `
			params = append(params, size)
			if page > 0 {
				sql += `offset ? `
				params = append(params, size*(page-1))
			}
		}
	}

	if _, err := GetOrmer().Raw(sql, params).QueryRows(&repositories); err != nil {
		return nil, err
	}

	return repositories, nil
}

func repositoryQueryConditions(query ...*models.RepositoryQuery) (string, []interface{}) {
	params := []interface{}{}
	sql := `from repository r `
	if len(query) == 0 || query[0] == nil {
		return sql, params
	}
	q := query[0]

	if q.LabelID > 0 {
		sql += `join harbor_resource_label rl on r.repository_id = rl.resource_id
		and rl.resource_type = 'r' `
	}
	sql += `where 1=1 `

	if len(q.Name) > 0 {
		sql += `and r.name like ? `
		params = append(params, "%"+Escape(q.Name)+"%")
	}

	if len(q.ProjectIDs) > 0 {
		sql += fmt.Sprintf(`and r.project_id in ( %s ) `,
			paramPlaceholder(len(q.ProjectIDs)))
		params = append(params, q.ProjectIDs)
	}

	if len(q.ProjectName) > 0 {
		// use "like" rather than "table joining" because that
		// in integration mode the projects are saved in Admiral side
		sql += `and r.name like ? `
		params = append(params, q.ProjectName+"/%")
	}

	if q.LabelID > 0 {
		sql += `and rl.label_id = ? `
		params = append(params, q.LabelID)
	}

	return sql, params
}
<commit_msg>fix gofmt<commit_after>// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dao

import (
	"fmt"
	"time"

	"github.com/astaxie/beego/orm"
	"github.com/goharbor/harbor/src/common/models"
)

var orderMap = map[string]string{
	"name":           "name asc",
	"+name":          "name asc",
	"-name":          "name desc",
	"creation_time":  "creation_time asc",
	"+creation_time": "creation_time asc",
	"-creation_time": "creation_time desc",
	"update_time":    "update_time asc",
	"+update_time":   "update_time asc",
	"-update_time":   "update_time desc",
}

// AddRepository adds a repo to the database.
func AddRepository(repo models.RepoRecord) error {
	if repo.ProjectID == 0 {
		return fmt.Errorf("invalid project ID: %d", repo.ProjectID)
	}

	o := GetOrmer()
	now := time.Now()
	repo.CreationTime = now
	repo.UpdateTime = now
	_, err := o.Insert(&repo)
	return err
}

// GetRepositoryByName ...
func GetRepositoryByName(name string) (*models.RepoRecord, error) {
	o := GetOrmer()
	r := models.RepoRecord{Name: name}
	err := o.Read(&r, "Name")
	if err == orm.ErrNoRows {
		return nil, nil
	}
	return &r, err
}

// DeleteRepository ...
func DeleteRepository(name string) error {
	o := GetOrmer()
	_, err := o.QueryTable("repository").Filter("name", name).Delete()
	return err
}

// UpdateRepository ...
func UpdateRepository(repo models.RepoRecord) error {
	o := GetOrmer()
	repo.UpdateTime = time.Now()
	_, err := o.Update(&repo)
	return err
}

// IncreasePullCount ...
func IncreasePullCount(name string) (err error) {
	o := GetOrmer()
	num, err := o.QueryTable("repository").Filter("name", name).Update(
		orm.Params{
			"pull_count":  orm.ColValue(orm.ColAdd, 1),
			"update_time": time.Now(),
		})
	if err != nil {
		return err
	}
	if num == 0 {
		return fmt.Errorf("Failed to increase repository pull count with name: %s", name)
	}
	return nil
}

//RepositoryExists returns whether the repository exists according to its name.
func RepositoryExists(name string) bool {
	o := GetOrmer()
	return o.QueryTable("repository").Filter("name", name).Exist()
}

//GetTopRepos returns the most popular repositories whose project ID is
// in projectIDs
func GetTopRepos(projectIDs []int64, n int) ([]*models.RepoRecord, error) {
	repositories := []*models.RepoRecord{}
	if len(projectIDs) == 0 {
		return repositories, nil
	}

	_, err := GetOrmer().QueryTable(&models.RepoRecord{}).
		Filter("project_id__in", projectIDs).
		OrderBy("-pull_count").
		Limit(n).
		All(&repositories)

	return repositories, err
}

// GetTotalOfRepositories ...
func GetTotalOfRepositories(query ...*models.RepositoryQuery) (int64, error) {
	sql, params := repositoryQueryConditions(query...)
	sql = `select count(*) ` + sql
	var total int64
	if err := GetOrmer().Raw(sql, params).QueryRow(&total); err != nil {
		return 0, err
	}
	return total, nil
}

// GetRepositories ...
func GetRepositories(query ...*models.RepositoryQuery) ([]*models.RepoRecord, error) {
	repositories := []*models.RepoRecord{}
	order := "name asc"
	if len(query) > 0 && query[0] != nil {
		if s, ok := orderMap[query[0].Sort]; ok {
			order = s
		}
	}

	condition, params := repositoryQueryConditions(query...)
	sql := fmt.Sprintf(`select r.repository_id, r.name, r.project_id, r.description, r.pull_count,
	r.star_count, r.creation_time, r.update_time %s order by r.%s `, condition, order)
	if len(query) > 0 && query[0] != nil {
		page, size := query[0].Page, query[0].Size
		if size > 0 {
			sql += `limit ? `
			params = append(params, size)
			if page > 0 {
				sql += `offset ? `
				params = append(params, size*(page-1))
			}
		}
	}

	if _, err := GetOrmer().Raw(sql, params).QueryRows(&repositories); err != nil {
		return nil, err
	}

	return repositories, nil
}

func repositoryQueryConditions(query ...*models.RepositoryQuery) (string, []interface{}) {
	params := []interface{}{}
	sql := `from repository r `
	if len(query) == 0 || query[0] == nil {
		return sql, params
	}
	q := query[0]

	if q.LabelID > 0 {
		sql += `join harbor_resource_label rl on r.repository_id = rl.resource_id
		and rl.resource_type = 'r' `
	}
	sql += `where 1=1 `

	if len(q.Name) > 0 {
		sql += `and r.name like ? `
		params = append(params, "%"+Escape(q.Name)+"%")
	}

	if len(q.ProjectIDs) > 0 {
		sql += fmt.Sprintf(`and r.project_id in ( %s ) `,
			paramPlaceholder(len(q.ProjectIDs)))
		params = append(params, q.ProjectIDs)
	}

	if len(q.ProjectName) > 0 {
		// use "like" rather than "table joining" because that
		// in integration mode the projects are saved in Admiral side
		sql += `and r.name like ? `
		params = append(params, q.ProjectName+"/%")
	}

	if q.LabelID > 0 {
		sql += `and rl.label_id = ? `
		params = append(params, q.LabelID)
	}

	return sql, params
}
<|endoftext|>


<commit_before>package job

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/antlinker/sdk/asapi"
	"github.com/antlinker/sdk/utils"
)

const (
	jobRouter = "/job/exec"
)

// Config 配置参数
type Config struct {
	HTTPAddr string
}

// NewHandle 创建作业任务
func NewHandle(auh *asapi.AuthorizeHandle, config *Config) *Handle {
	return &Handle{
		auh: auh,
		cfg: config,
	}
}

// Handle 作业任务
type Handle struct {
	auh *asapi.AuthorizeHandle
	cfg *Config
}

func (h *Handle) getURL(router string) string {
	addr := h.cfg.HTTPAddr
	if len(addr) == 0 {
		return ""
	}

	if addr[len(addr)-1] == '/' {
		addr = addr[:len(addr)-1]
	}

	return addr + router
}

func (h *Handle) getANTUserID(intelUserCode string) (userID string, err error) {
	auids, ar := h.auh.GetAntUIDList("", intelUserCode)
	if ar != nil {
		err = ar
		return
	} else if len(auids) == 0 {
		err = errors.New("not found user")
		return
	}
	userID = auids[0]
	return
}

func (h *Handle) request(body interface{}) (err error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)

	defer cancel()

	data, err := utils.PostJSON(ctx, h.getURL(jobRouter), body, func(req *http.Request) (*http.Request, error) {
		token, err := asapi.GetToken()
		if err != nil {
			return nil, err
		}
		req.Header.Set("AccessToken", token)
		return req, nil
	})
	if err != nil {
		if len(data) > 0 {
			log.Println("请求发生错误：", string(data))
		}
		return
	}

	var str string
	json.Unmarshal(data, str)
	if str != "ok" {
		err = fmt.Errorf(string(data))
	}

	return
}

// ModifyStaffName 学工更改用户姓名
func (h *Handle) ModifyStaffName(intelUserCode, name string) (err error) {
	userID, err := h.getANTUserID(intelUserCode)
	if err != nil {
		return
	}

	body := map[string]string{
		"UID":  userID,
		"Name": name,
	}

	err = h.request(body)

	return
}

// ModifyStaffDept 学工更改学工用户所在的部门
func (h *Handle) ModifyStaffDept(intelUserCode, deptID, deptName string) (err error) {
	userID, err := h.getANTUserID(intelUserCode)
	if err != nil {
		return
	}

	body := map[string]string{
		"UID":      userID,
		"DeptID":   deptID,
		"DeptName": deptName,
	}

	err = h.request(body)

	return
}

// ModifyStudentClassRequest 学工更改学生用户所在的班级请求参数
type ModifyStudentClassRequest struct {
	AcademyID   string
	AcademyName string
	MajorID     string
	MajorName   string
	GradeName   string
	ClassID     string
	ClassName   string
}

// ModifyStudentClass 学工更改学生用户所在的班级
func (h *Handle) ModifyStudentClass(intelUserCode string, req *ModifyStudentClassRequest) (err error) {
	userID, err := h.getANTUserID(intelUserCode)
	if err != nil {
		return
	}

	body := map[string]string{
		"UID":         userID,
		"AcademyID":   req.AcademyID,
		"AcademyName": req.AcademyName,
		"MajorID":     req.MajorID,
		"MajorName":   req.MajorName,
		"GradeName":   req.GradeName,
		"ClassID":     req.ClassID,
		"ClassName":   req.ClassName,
	}

	err = h.request(body)

	return
}

// ModifyStaffClass 更改辅导员用户所管理的班级
func (h *Handle) ModifyStaffClass(intelUserCode string) (err error) {
	userID, err := h.getANTUserID(intelUserCode)
	if err != nil {
		return
	}

	body := map[string]string{
		"UID": userID,
	}

	err = h.request(body)

	return
}

// ModifyStaffDeptName 更改学工用户所在的部门名称
func (h *Handle) ModifyStaffDeptName(university, deptID, deptName string) (err error) {
	body := map[string]string{
		"University": university,
		"DeptID":     deptID,
		"DeptName":   deptName,
	}

	err = h.request(body)

	return
}

// ModifyStaffClassName 更改学工用户所在的班级名称
func (h *Handle) ModifyStaffClassName(university, deptID, deptName string) (err error) {
	body := map[string]string{
		"University": university,
		"DeptID":     deptID,
		"DeptName":   deptName,
	}

	err = h.request(body)

	return
}
<commit_msg>更新job<commit_after>package job

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/antlinker/sdk/asapi"
	"github.com/antlinker/sdk/utils"
)

const (
	jobRouter = "/job/exec"
)

// Config 配置参数
type Config struct {
	HTTPAddr string
}

// NewHandle 创建作业任务
func NewHandle(auh *asapi.AuthorizeHandle, config *Config) *Handle {
	return &Handle{
		auh: auh,
		cfg: config,
	}
}

// Handle 作业任务
type Handle struct {
	auh *asapi.AuthorizeHandle
	cfg *Config
}

func (h *Handle) getURL(router string) string {
	addr := h.cfg.HTTPAddr
	if len(addr) == 0 {
		return ""
	}

	if addr[len(addr)-1] == '/' {
		addr = addr[:len(addr)-1]
	}

	return addr + router
}

func (h *Handle) getANTUserID(intelUserCode string) (userID string, err error) {
	auids, ar := h.auh.GetAntUIDList("", intelUserCode)
	if ar != nil {
		err = ar
		return
	} else if len(auids) == 0 {
		err = errors.New("not found user")
		return
	}
	userID = auids[0]
	return
}

func (h *Handle) request(body interface{}) (err error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)

	defer cancel()

	data, err := utils.PostJSON(ctx, h.getURL(jobRouter), body, func(req *http.Request) (*http.Request, error) {
		token, err := asapi.GetToken()
		if err != nil {
			return nil, err
		}
		req.Header.Set("AccessToken", token)
		return req, nil
	})
	if err != nil {
		if len(data) > 0 {
			log.Println("请求发生错误：", string(data))
		}
		return
	}

	var str string
	json.Unmarshal(data, &str)
	if str != "ok" {
		err = fmt.Errorf(string(data))
	}

	return
}

// ModifyStaffName 学工更改用户姓名
func (h *Handle) ModifyStaffName(intelUserCode, name string) (err error) {
	userID, err := h.getANTUserID(intelUserCode)
	if err != nil {
		return
	}

	body := map[string]string{
		"UID":  userID,
		"Name": name,
	}

	err = h.request(body)

	return
}

// ModifyStaffDept 学工更改学工用户所在的部门
func (h *Handle) ModifyStaffDept(intelUserCode, deptID, deptName string) (err error) {
	userID, err := h.getANTUserID(intelUserCode)
	if err != nil {
		return
	}

	body := map[string]string{
		"UID":      userID,
		"DeptID":   deptID,
		"DeptName": deptName,
	}

	err = h.request(body)

	return
}

// ModifyStudentClassRequest 学工更改学生用户所在的班级请求参数
type ModifyStudentClassRequest struct {
	AcademyID   string
	AcademyName string
	MajorID     string
	MajorName   string
	GradeName   string
	ClassID     string
	ClassName   string
}

// ModifyStudentClass 学工更改学生用户所在的班级
func (h *Handle) ModifyStudentClass(intelUserCode string, req *ModifyStudentClassRequest) (err error) {
	userID, err := h.getANTUserID(intelUserCode)
	if err != nil {
		return
	}

	body := map[string]string{
		"UID":         userID,
		"AcademyID":   req.AcademyID,
		"AcademyName": req.AcademyName,
		"MajorID":     req.MajorID,
		"MajorName":   req.MajorName,
		"GradeName":   req.GradeName,
		"ClassID":     req.ClassID,
		"ClassName":   req.ClassName,
	}

	err = h.request(body)

	return
}

// ModifyStaffClass 更改辅导员用户所管理的班级
func (h *Handle) ModifyStaffClass(intelUserCode string) (err error) {
	userID, err := h.getANTUserID(intelUserCode)
	if err != nil {
		return
	}

	body := map[string]string{
		"UID": userID,
	}

	err = h.request(body)

	return
}

// ModifyStaffDeptName 更改学工用户所在的部门名称
func (h *Handle) ModifyStaffDeptName(university, deptID, deptName string) (err error) {
	body := map[string]string{
		"University": university,
		"DeptID":     deptID,
		"DeptName":   deptName,
	}

	err = h.request(body)

	return
}

// ModifyStaffClassName 更改学工用户所在的班级名称
func (h *Handle) ModifyStaffClassName(university, deptID, deptName string) (err error) {
	body := map[string]string{
		"University": university,
		"DeptID":     deptID,
		"DeptName":   deptName,
	}

	err = h.request(body)

	return
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package monitor

import (
	"context"
	"fmt"
	"time"

	"github.com/google/keytransparency/core/client"
	"github.com/google/keytransparency/core/monitorstorage"
	"github.com/google/trillian"

	"github.com/google/trillian/types"

	"github.com/golang/glog"

	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
	tclient "github.com/google/trillian/client"
	tcrypto "github.com/google/trillian/crypto"
)

// Monitor holds the internal state for a monitor accessing the mutations API
// and for verifying its responses.
type Monitor struct {
	cli         *client.Client
	logVerifier *tclient.LogVerifier
	mapVerifier *tclient.MapVerifier
	signer      *tcrypto.Signer
	store       monitorstorage.Interface
}

// NewFromDirectory produces a new monitor from a Directory object.
func NewFromDirectory(cli pb.KeyTransparencyClient,
	config *pb.Directory,
	signer *tcrypto.Signer,
	store monitorstorage.Interface) (*Monitor, error) {
	logVerifier, err := tclient.NewLogVerifierFromTree(config.GetLog())
	if err != nil {
		return nil, fmt.Errorf("could not initialize log verifier: %v", err)
	}
	mapVerifier, err := tclient.NewMapVerifierFromTree(config.GetMap())
	if err != nil {
		return nil, fmt.Errorf("could not initialize map verifier: %v", err)
	}

	ktClient, err := client.NewFromConfig(cli, config)
	if err != nil {
		return nil, fmt.Errorf("could not create kt client: %v", err)
	}

	return New(ktClient, logVerifier, mapVerifier, signer, store)
}

// New creates a new instance of the monitor.
func New(cli *client.Client,
	logVerifier *tclient.LogVerifier,
	mapVerifier *tclient.MapVerifier,
	signer *tcrypto.Signer,
	store monitorstorage.Interface) (*Monitor, error) {
	return &Monitor{
		cli:         cli,
		logVerifier: logVerifier,
		mapVerifier: mapVerifier,
		signer:      signer,
		store:       store,
	}, nil
}

// RevisionPair is two adjacent revisions.
type RevisionPair struct {
	A, B *types.MapRootV1
}

// RevisionPairs consumes revisions (0, 1, 2) and produces pairs (0,1), (1,2).
func RevisionPairs(ctx context.Context, revisions <-chan *types.MapRootV1, pairs chan<- RevisionPair) error {
	defer close(pairs)
	var revisionA *types.MapRootV1
	for revision := range revisions {
		if revisionA == nil {
			revisionA = revision
			continue
		}
		pair := RevisionPair{
			A: revisionA,
			B: revision,
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case pairs <- pair:
		}
		revisionA = revision
	}
	return nil
}

// ProcessLoop continuously fetches mutations and processes them.
func (m *Monitor) ProcessLoop(ctx context.Context, startRev int64) error {
	cctx, cancel := context.WithCancel(ctx)
	errc := make(chan error)
	revisions := make(chan *types.MapRootV1)
	pairs := make(chan RevisionPair)

	go func(ctx context.Context) {
		err := m.cli.StreamRevisions(ctx, startRev, revisions)
		glog.Errorf("StreamRevisions(%v): %v", startRev, err)
		errc <- err
	}(cctx)
	go func(ctx context.Context) {
		err := RevisionPairs(ctx, revisions, pairs)
		glog.Errorf("RevisionPairs(): %v", err)
		errc <- err
	}(cctx)
	defer cancel()

	for pair := range pairs {
		mutations, err := m.cli.RevisionMutations(ctx, pair.B)
		if err != nil {
			return err
		}

		var smr *trillian.SignedMapRoot
		var errList []error

		if errs := m.verifyMutations(mutations, pair.A, pair.B); len(errs) > 0 {
			glog.Errorf("Invalid Revision %v Mutations: %v", pair.B.Revision, errs)
			errList = errs
		} else {
			// Sign if successful.
			smr, err = m.signer.SignMapRoot(pair.B)
			if err != nil {
				return err
			}
		}

		// Save result.
		if err := m.store.Set(int64(pair.B.Revision), &monitorstorage.Result{
			Smr:    smr,
			Seen:   time.Now(),
			Errors: errList,
		}); err != nil {
			return fmt.Errorf("monitorstorage.Set(%v, _): %v", pair.B.Revision, err)
		}
	}
	errA := <-errc
	errB := <-errc
	if err := errA; err != nil {
		return err
	}
	return errB
}
<commit_msg>Remove unused logVerifier (#1324)<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package monitor

import (
	"context"
	"fmt"
	"time"

	"github.com/google/keytransparency/core/client"
	"github.com/google/keytransparency/core/monitorstorage"
	"github.com/google/trillian"

	"github.com/google/trillian/types"

	"github.com/golang/glog"

	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
	tclient "github.com/google/trillian/client"
	tcrypto "github.com/google/trillian/crypto"
)

// Monitor holds the internal state for a monitor accessing the mutations API
// and for verifying its responses.
type Monitor struct {
	cli         *client.Client
	mapVerifier *tclient.MapVerifier
	signer      *tcrypto.Signer
	store       monitorstorage.Interface
}

// NewFromDirectory produces a new monitor from a Directory object.
func NewFromDirectory(cli pb.KeyTransparencyClient,
	config *pb.Directory,
	signer *tcrypto.Signer,
	store monitorstorage.Interface) (*Monitor, error) {
	mapVerifier, err := tclient.NewMapVerifierFromTree(config.GetMap())
	if err != nil {
		return nil, fmt.Errorf("could not initialize map verifier: %v", err)
	}

	ktClient, err := client.NewFromConfig(cli, config)
	if err != nil {
		return nil, fmt.Errorf("could not create kt client: %v", err)
	}

	return New(ktClient, mapVerifier, signer, store)
}

// New creates a new instance of the monitor.
func New(cli *client.Client,
	mapVerifier *tclient.MapVerifier,
	signer *tcrypto.Signer,
	store monitorstorage.Interface) (*Monitor, error) {
	return &Monitor{
		cli:         cli,
		mapVerifier: mapVerifier,
		signer:      signer,
		store:       store,
	}, nil
}

// RevisionPair is two adjacent revisions.
type RevisionPair struct {
	A, B *types.MapRootV1
}

// RevisionPairs consumes revisions (0, 1, 2) and produces pairs (0,1), (1,2).
func RevisionPairs(ctx context.Context, revisions <-chan *types.MapRootV1, pairs chan<- RevisionPair) error {
	defer close(pairs)
	var revisionA *types.MapRootV1
	for revision := range revisions {
		if revisionA == nil {
			revisionA = revision
			continue
		}
		pair := RevisionPair{
			A: revisionA,
			B: revision,
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case pairs <- pair:
		}
		revisionA = revision
	}
	return nil
}

// ProcessLoop continuously fetches mutations and processes them.
func (m *Monitor) ProcessLoop(ctx context.Context, startRev int64) error {
	cctx, cancel := context.WithCancel(ctx)
	errc := make(chan error)
	revisions := make(chan *types.MapRootV1)
	pairs := make(chan RevisionPair)

	go func(ctx context.Context) {
		err := m.cli.StreamRevisions(ctx, startRev, revisions)
		glog.Errorf("StreamRevisions(%v): %v", startRev, err)
		errc <- err
	}(cctx)
	go func(ctx context.Context) {
		err := RevisionPairs(ctx, revisions, pairs)
		glog.Errorf("RevisionPairs(): %v", err)
		errc <- err
	}(cctx)
	defer cancel()

	for pair := range pairs {
		mutations, err := m.cli.RevisionMutations(ctx, pair.B)
		if err != nil {
			return err
		}

		var smr *trillian.SignedMapRoot
		var errList []error

		if errs := m.verifyMutations(mutations, pair.A, pair.B); len(errs) > 0 {
			glog.Errorf("Invalid Revision %v Mutations: %v", pair.B.Revision, errs)
			errList = errs
		} else {
			// Sign if successful.
			smr, err = m.signer.SignMapRoot(pair.B)
			if err != nil {
				return err
			}
		}

		// Save result.
		if err := m.store.Set(int64(pair.B.Revision), &monitorstorage.Result{
			Smr:    smr,
			Seen:   time.Now(),
			Errors: errList,
		}); err != nil {
			return fmt.Errorf("monitorstorage.Set(%v, _): %v", pair.B.Revision, err)
		}
	}
	errA := <-errc
	errB := <-errc
	if err := errA; err != nil {
		return err
	}
	return errB
}
<|endoftext|>


<commit_before>package service

import (
	"log"
	"os"

	"github.com/coreos/go-etcd/etcd"
)

var (
	logger   *log.Logger
	client   *etcd.Client
	hostname string

	cacert  = os.Getenv("ETCD_CACERT")
	tlskey  = os.Getenv("ETCD_TLS_KEY")
	tlscert = os.Getenv("ETCD_TLS_CERT")
)

func init() {
	host := "http://localhost:4001"
	if len(os.Getenv("ETCD_HOST")) != 0 {
		host = os.Getenv("ETCD_HOST")
	}
	if len(cacert) != 0 && len(tlskey) != 0 && len(tlscert) != 0 {
		client = newTLSClient([]string{host})
	} else {
		client = etcd.NewClient([]string{host})
	}

	if len(os.Getenv("HOSTNAME")) != 0 {
		hostname = os.Getenv("HOSTNAME")
	} else {
		h, err := os.Hostname()
		if err != nil {
			panic(err)
		}
		hostname = h
	}

	logger = log.New(os.Stderr, "[etcd-discovery]", log.LstdFlags)
}

func newTLSClient(hosts []string) *etcd.Client {
	c, err := etcd.NewTLSClient(hosts, tlscert, tlskey, cacert)
	if err != nil {
		panic(err)
	}
	return c
}
<commit_msg>Use https URL if using TLS<commit_after>package service

import (
	"log"
	"os"
	"strings"

	"github.com/coreos/go-etcd/etcd"
)

var (
	logger   *log.Logger
	client   *etcd.Client
	hostname string

	cacert  = os.Getenv("ETCD_CACERT")
	tlskey  = os.Getenv("ETCD_TLS_KEY")
	tlscert = os.Getenv("ETCD_TLS_CERT")
)

func init() {
	host := "http://localhost:4001"
	if len(os.Getenv("ETCD_HOST")) != 0 {
		host = os.Getenv("ETCD_HOST")
	}
	if len(cacert) != 0 && len(tlskey) != 0 && len(tlscert) != 0 {
		if !strings.Contains(host, "https://") {
			host = strings.Replace(host, "http", "https", 1)
		}
		client = newTLSClient([]string{host})
	} else {
		client = etcd.NewClient([]string{host})
	}

	if len(os.Getenv("HOSTNAME")) != 0 {
		hostname = os.Getenv("HOSTNAME")
	} else {
		h, err := os.Hostname()
		if err != nil {
			panic(err)
		}
		hostname = h
	}

	logger = log.New(os.Stderr, "[etcd-discovery]", log.LstdFlags)
}

func newTLSClient(hosts []string) *etcd.Client {
	c, err := etcd.NewTLSClient(hosts, tlscert, tlskey, cacert)
	if err != nil {
		panic(err)
	}
	return c
}
<|endoftext|>


<commit_before>package chunk

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"sync"

	"time"

	. "github.com/claudetech/loggo/default"
)

type Storage struct {
	ChunkPath  string
	ChunkSize  int64
	MaxChunks  int
	queue      chan *Item
	chunks     map[string][]byte
	chunksLock sync.Mutex
	toc        map[string]bool
	tocLock    sync.Mutex
	stack      *Stack
}

type Item struct {
	id    string
	bytes []byte
}

func NewStorage(chunkPath string, chunkSize int64, maxChunks int) *Storage {
	storage := Storage{
		ChunkPath: chunkPath,
		ChunkSize: chunkSize,
		MaxChunks: maxChunks,
		queue:     make(chan *Item, 100),
		chunks:    make(map[string][]byte),
		toc:       make(map[string]bool),
		stack:     NewStack(),
	}

	go storage.thread()

	return &storage
}

func (s *Storage) Clear() error {
	if err := os.RemoveAll(s.ChunkPath); nil != err {
		return fmt.Errorf("Could not clear old chunks from disk")
	}
	return nil
}

func (s *Storage) ExistsOrCreate(id string) bool {
	s.tocLock.Lock()
	if _, exists := s.toc[id]; exists {
		s.tocLock.Unlock()
		return true
	}
	s.toc[id] = true
	s.tocLock.Unlock()
	return false
}

func (s *Storage) Store(id string, bytes []byte) error {
	s.chunksLock.Lock()
	s.chunks[id] = bytes
	s.chunksLock.Unlock()

	s.queue <- &Item{
		id:    id,
		bytes: bytes,
	}

	return nil
}

func (s *Storage) Get(id string, offset, size int64) ([]byte, error) {
	res := make(chan []byte)

	go func() {
		for {
			s.tocLock.Lock()
			_, exists := s.toc[id]
			s.tocLock.Unlock()
			if exists {
				bytes, exists := s.loadFromRAM(id, offset, size)
				if exists {
					res <- bytes
					return
				}

				bytes, exists = s.loadFromDisk(id, offset, size)
				if exists {
					res <- bytes
					return
				}
			}

			time.Sleep(10 * time.Millisecond)
		}
	}()

	return <-res, nil
}

func (s *Storage) thread() {
	for {
		item := <-s.queue
		if err := s.storeToDisk(item.id, item.bytes); nil != err {
			Log.Warningf("%v", err)
		}
	}
}

func (s *Storage) loadFromRAM(id string, offset, size int64) ([]byte, bool) {
	bytes, exists := s.chunks[id]
	if !exists {
		return nil, false
	}

	sOffset := int64(math.Min(float64(len(bytes)), float64(offset)))
	eOffset := int64(math.Min(float64(len(bytes)), float64(offset+size)))
	return bytes[sOffset:eOffset], true
}

func (s *Storage) loadFromDisk(id string, offset, size int64) ([]byte, bool) {
	filename := filepath.Join(s.ChunkPath, id)

	f, err := os.Open(filename)
	if nil != err {
		Log.Tracef("%v", err)
		return nil, false
	}
	defer f.Close()

	buf := make([]byte, size)
	n, err := f.ReadAt(buf, offset)
	if n > 0 && (nil == err || io.EOF == err || io.ErrUnexpectedEOF == err) {
		s.stack.Touch(id)

		eOffset := int64(math.Min(float64(size), float64(len(buf))))
		return buf[:eOffset], true
	}

	Log.Tracef("%v", err)
	return nil, false
}

func (s *Storage) storeToDisk(id string, bytes []byte) error {
	filename := filepath.Join(s.ChunkPath, id)

	if s.stack.Len() >= s.MaxChunks {
		deleteID := s.stack.Pop()

		if "" != deleteID {
			filename := filepath.Join(s.ChunkPath, deleteID)

			Log.Debugf("Deleting chunk %v", filename)
			if err := os.Remove(filename); nil != err {
				Log.Debugf("%v", err)
				Log.Warningf("Could not delete chunk %v", filename)
			}

			s.tocLock.Lock()
			delete(s.toc, deleteID)
			s.tocLock.Unlock()
		}
	}

	if _, err := os.Stat(s.ChunkPath); os.IsNotExist(err) {
		if err := os.MkdirAll(s.ChunkPath, 0777); nil != err {
			Log.Debugf("%v", err)
			return fmt.Errorf("Could not create chunk temp path %v", s.ChunkPath)
		}
	}

	if _, err := os.Stat(filename); os.IsNotExist(err) {
		if err := ioutil.WriteFile(filename, bytes, 0777); nil != err {
			Log.Debugf("%v", err)
			return fmt.Errorf("Could not write chunk temp file %v", filename)
		}

		s.stack.Push(id)
	}

	s.chunksLock.Lock()
	delete(s.chunks, id)
	s.chunksLock.Unlock()

	return nil
}
<commit_msg>chunk locking<commit_after>package chunk

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"sync"

	"time"

	. "github.com/claudetech/loggo/default"
)

type Storage struct {
	ChunkPath  string
	ChunkSize  int64
	MaxChunks  int
	queue      chan *Item
	chunks     map[string][]byte
	chunksLock sync.Mutex
	toc        map[string]bool
	tocLock    sync.Mutex
	stack      *Stack
}

type Item struct {
	id    string
	bytes []byte
}

func NewStorage(chunkPath string, chunkSize int64, maxChunks int) *Storage {
	storage := Storage{
		ChunkPath: chunkPath,
		ChunkSize: chunkSize,
		MaxChunks: maxChunks,
		queue:     make(chan *Item, 100),
		chunks:    make(map[string][]byte),
		toc:       make(map[string]bool),
		stack:     NewStack(),
	}

	go storage.thread()

	return &storage
}

func (s *Storage) Clear() error {
	if err := os.RemoveAll(s.ChunkPath); nil != err {
		return fmt.Errorf("Could not clear old chunks from disk")
	}
	return nil
}

func (s *Storage) ExistsOrCreate(id string) bool {
	s.tocLock.Lock()
	if _, exists := s.toc[id]; exists {
		s.tocLock.Unlock()
		return true
	}
	s.toc[id] = true
	s.tocLock.Unlock()
	return false
}

func (s *Storage) Store(id string, bytes []byte) error {
	s.chunksLock.Lock()
	s.chunks[id] = bytes
	s.chunksLock.Unlock()

	s.queue <- &Item{
		id:    id,
		bytes: bytes,
	}

	return nil
}

func (s *Storage) Get(id string, offset, size int64) ([]byte, error) {
	res := make(chan []byte)

	go func() {
		for {
			s.tocLock.Lock()
			_, exists := s.toc[id]
			s.tocLock.Unlock()
			if exists {
				bytes, exists := s.loadFromRAM(id, offset, size)
				if exists {
					res <- bytes
					return
				}

				bytes, exists = s.loadFromDisk(id, offset, size)
				if exists {
					res <- bytes
					return
				}
			}

			time.Sleep(10 * time.Millisecond)
		}
	}()

	return <-res, nil
}

func (s *Storage) thread() {
	for {
		item := <-s.queue
		if err := s.storeToDisk(item.id, item.bytes); nil != err {
			Log.Warningf("%v", err)
		}
	}
}

func (s *Storage) loadFromRAM(id string, offset, size int64) ([]byte, bool) {
	s.chunksLock.Lock()
	bytes, exists := s.chunks[id]
	s.chunksLock.Unlock()
	if !exists {
		return nil, false
	}

	sOffset := int64(math.Min(float64(len(bytes)), float64(offset)))
	eOffset := int64(math.Min(float64(len(bytes)), float64(offset+size)))
	return bytes[sOffset:eOffset], true
}

func (s *Storage) loadFromDisk(id string, offset, size int64) ([]byte, bool) {
	filename := filepath.Join(s.ChunkPath, id)

	f, err := os.Open(filename)
	if nil != err {
		Log.Tracef("%v", err)
		return nil, false
	}
	defer f.Close()

	buf := make([]byte, size)
	n, err := f.ReadAt(buf, offset)
	if n > 0 && (nil == err || io.EOF == err || io.ErrUnexpectedEOF == err) {
		s.stack.Touch(id)

		eOffset := int64(math.Min(float64(size), float64(len(buf))))
		return buf[:eOffset], true
	}

	Log.Tracef("%v", err)
	return nil, false
}

func (s *Storage) storeToDisk(id string, bytes []byte) error {
	filename := filepath.Join(s.ChunkPath, id)

	if s.stack.Len() >= s.MaxChunks {
		deleteID := s.stack.Pop()

		if "" != deleteID {
			filename := filepath.Join(s.ChunkPath, deleteID)

			Log.Debugf("Deleting chunk %v", filename)
			if err := os.Remove(filename); nil != err {
				Log.Debugf("%v", err)
				Log.Warningf("Could not delete chunk %v", filename)
			}

			s.tocLock.Lock()
			delete(s.toc, deleteID)
			s.tocLock.Unlock()
		}
	}

	if _, err := os.Stat(s.ChunkPath); os.IsNotExist(err) {
		if err := os.MkdirAll(s.ChunkPath, 0777); nil != err {
			Log.Debugf("%v", err)
			return fmt.Errorf("Could not create chunk temp path %v", s.ChunkPath)
		}
	}

	if _, err := os.Stat(filename); os.IsNotExist(err) {
		if err := ioutil.WriteFile(filename, bytes, 0777); nil != err {
			Log.Debugf("%v", err)
			return fmt.Errorf("Could not write chunk temp file %v", filename)
		}

		s.stack.Push(id)
	}

	s.chunksLock.Lock()
	delete(s.chunks, id)
	s.chunksLock.Unlock()

	return nil
}
<|endoftext|>


<commit_before>// Package crowdauth provides middleware for Crowd SSO logins
//
// Goals:
//  1) drop-in authentication against Crowd SSO
//  2) make it easy to use Crowd SSO as part of your own auth flow
package crowdauth // import "go.jona.me/crowd/crowdauth"

import (
	"go.jona.me/crowd"
	"html/template"
	"log"
	"net/http"
	"time"
)

type SSO struct {
	CrowdApp            *crowd.Crowd
	LoginPage           AuthLoginPage
	LoginTemplate       *template.Template
	ClientAddressFinder ClientAddressFinder
	CookieConfig        crowd.CookieConfig
}

// The AuthLoginPage type extends the normal http.HandlerFunc type
// with a boolean return to indicate login success or failure.
type AuthLoginPage func(http.ResponseWriter, *http.Request, *SSO) bool

// ClientAddressFinder type represents a function that returns the
// end-user's IP address, allowing you to handle cases where the address
// is masked by proxy servers etc by checking headers or whatever to find
// the user's address
type ClientAddressFinder func(*http.Request) (string, error)

var authErr string = "unauthorized, login required"

func DefaultClientAddressFinder(r *http.Request) (addr string, err error) {
	return r.RemoteAddr, nil
}

// New creates a new instance of SSO
func New(user string, password string, crowdURL string) (s *SSO, err error) {
	s = &SSO{}
	s.LoginPage = loginPage
	s.ClientAddressFinder = DefaultClientAddressFinder
	s.LoginTemplate = template.Must(template.New("authPage").Parse(defLoginPage))

	cwd, err := crowd.New(user, password, crowdURL)
	if err != nil {
		return s, err
	}
	s.CrowdApp = &cwd
	s.CookieConfig, err = s.CrowdApp.GetCookieConfig()
	if err != nil {
		return s, err
	}

	return s, nil
}

// Handler provides HTTP middleware using http.Handler chaining
// that requires user authentication via Atlassian Crowd SSO.
func (s *SSO) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if s.loginHandler(w, r) == false {
			return
		}
		h.ServeHTTP(w, r)
	})

}

func (s *SSO) loginHandler(w http.ResponseWriter, r *http.Request) bool {
	ck, err := r.Cookie(s.CookieConfig.Name)

	if err == http.ErrNoCookie {
		// no cookie so show login page if GET
		// if POST check if login and handle
		// if fail, show login page with message
		if r.Method == "GET" {
			s.LoginPage(w, r, s)
		} else if r.Method == "POST" {
			authOK := s.LoginPage(w, r, s)
			if authOK == true {
				// Redirect for fresh pass through auth etc on success
				http.Redirect(w, r, r.RequestURI, http.StatusTemporaryRedirect)
				return false
			} else {
				log.Printf("crowdauth: authentication failed\n")
			}
		} else {
			http.Error(w, authErr, http.StatusUnauthorized)
		}
		return false
	} else {
		// validate cookie or show login page
		host, err := s.ClientAddressFinder(r)
		if err != nil {
			log.Printf("crowdauth: could not get remote addr: %s\n", err)
			return false
		}

		_, err = s.CrowdApp.ValidateSession(ck.Value, host)
		if err != nil {
			log.Printf("crowdauth: could not validate cookie, deleting because: %s\n", err)
			s.EndSession(w, r)
			s.LoginPage(w, r, s)
			return false
		}

		// valid cookie so fallthrough
	}
	return true
}

func (s *SSO) Login(user string, pass string, addr string) (cs crowd.Session, err error) {
	cs, err = s.CrowdApp.NewSession(user, pass, addr)
	return cs, err
}

func (s *SSO) Logout(w http.ResponseWriter, r *http.Request, newURL string) {
	s.EndSession(w, r)
	http.Redirect(w, r, newURL, http.StatusTemporaryRedirect)
}

// StartSession sets a Crowd session cookie.
func (s *SSO) StartSession(w http.ResponseWriter, cs crowd.Session) {
	ck := http.Cookie{
		Name:    s.CookieConfig.Name,
		Domain:  s.CookieConfig.Domain,
		Secure:  s.CookieConfig.Secure,
		Value:   cs.Token,
		Expires: cs.Expires,
	}
	http.SetCookie(w, &ck)
}

// EndSession invalidates the current Crowd session and cookie
func (s *SSO) EndSession(w http.ResponseWriter, r *http.Request) {
	currentCookie, _ := r.Cookie(s.CookieConfig.Name)
	newCookie := &http.Cookie{
		Name:    s.CookieConfig.Name,
		Domain:  s.CookieConfig.Domain,
		Secure:  s.CookieConfig.Secure,
		MaxAge:  -1,
		Expires: time.Unix(1, 0),
		Value:   "LOGGED-OUT",
	}
	s.CrowdApp.InvalidateSession(currentCookie.Value)

	log.Printf("Got cookie to remove: %+v\n", currentCookie)
	log.Printf("Removal cookie: %+v\n", newCookie)
	http.SetCookie(w, newCookie)
}

func loginPage(w http.ResponseWriter, r *http.Request, s *SSO) bool {
	if r.Method == "GET" { // show login page and bail
		showLoginPage(w, s)
		return false
	} else if r.Method == "POST" {
		user := r.FormValue("username")
		pass := r.FormValue("password")
		host, err := s.ClientAddressFinder(r)
		if err != nil {
			log.Printf("crowdauth: could not get remote addr: %s\n", err)
			showLoginPage(w, s)
			return false
		}

		sess, err := s.Login(user, pass, host)
		if err != nil {
			log.Printf("crowdauth: login/new session failed: %s\n", err)
			showLoginPage(w, s)
			return false
		}

		s.StartSession(w, sess)
	} else {
		return false
	}

	return true
}

func showLoginPage(w http.ResponseWriter, s *SSO) {
	err := s.LoginTemplate.ExecuteTemplate(w, "authPage", nil)
	if err != nil {
		log.Printf("crowdauth: could not exec template: %s\n", err)
	}
}
<commit_msg>Added GetUser, a function for middleware to fetch user information when a SSO session already exists.<commit_after>// Package crowdauth provides middleware for Crowd SSO logins
//
// Goals:
//  1) drop-in authentication against Crowd SSO
//  2) make it easy to use Crowd SSO as part of your own auth flow
package crowdauth // import "go.jona.me/crowd/crowdauth"

import (
	"errors"
	"go.jona.me/crowd"
	"html/template"
	"log"
	"net/http"
	"time"
)

type SSO struct {
	CrowdApp            *crowd.Crowd
	LoginPage           AuthLoginPage
	LoginTemplate       *template.Template
	ClientAddressFinder ClientAddressFinder
	CookieConfig        crowd.CookieConfig
}

// The AuthLoginPage type extends the normal http.HandlerFunc type
// with a boolean return to indicate login success or failure.
type AuthLoginPage func(http.ResponseWriter, *http.Request, *SSO) bool

// ClientAddressFinder type represents a function that returns the
// end-user's IP address, allowing you to handle cases where the address
// is masked by proxy servers etc by checking headers or whatever to find
// the user's address
type ClientAddressFinder func(*http.Request) (string, error)

var authErr string = "unauthorized, login required"

func DefaultClientAddressFinder(r *http.Request) (addr string, err error) {
	return r.RemoteAddr, nil
}

// New creates a new instance of SSO
func New(user string, password string, crowdURL string) (s *SSO, err error) {
	s = &SSO{}
	s.LoginPage = loginPage
	s.ClientAddressFinder = DefaultClientAddressFinder
	s.LoginTemplate = template.Must(template.New("authPage").Parse(defLoginPage))

	cwd, err := crowd.New(user, password, crowdURL)
	if err != nil {
		return s, err
	}
	s.CrowdApp = &cwd
	s.CookieConfig, err = s.CrowdApp.GetCookieConfig()
	if err != nil {
		return s, err
	}

	return s, nil
}

// Handler provides HTTP middleware using http.Handler chaining
// that requires user authentication via Atlassian Crowd SSO.
func (s *SSO) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if s.loginHandler(w, r) == false {
			return
		}
		h.ServeHTTP(w, r)
	})

}

func (s *SSO) loginHandler(w http.ResponseWriter, r *http.Request) bool {
	ck, err := r.Cookie(s.CookieConfig.Name)

	if err == http.ErrNoCookie {
		// no cookie so show login page if GET
		// if POST check if login and handle
		// if fail, show login page with message
		if r.Method == "GET" {
			s.LoginPage(w, r, s)
		} else if r.Method == "POST" {
			authOK := s.LoginPage(w, r, s)
			if authOK == true {
				// Redirect for fresh pass through auth etc on success
				http.Redirect(w, r, r.RequestURI, http.StatusTemporaryRedirect)
				return false
			} else {
				log.Printf("crowdauth: authentication failed\n")
			}
		} else {
			http.Error(w, authErr, http.StatusUnauthorized)
		}
		return false
	} else {
		// validate cookie or show login page
		host, err := s.ClientAddressFinder(r)
		if err != nil {
			log.Printf("crowdauth: could not get remote addr: %s\n", err)
			return false
		}

		_, err = s.CrowdApp.ValidateSession(ck.Value, host)
		if err != nil {
			log.Printf("crowdauth: could not validate cookie, deleting because: %s\n", err)
			s.EndSession(w, r)
			s.LoginPage(w, r, s)
			return false
		}

		// valid cookie so fallthrough
	}
	return true
}

func (s *SSO) Login(user string, pass string, addr string) (cs crowd.Session, err error) {
	cs, err = s.CrowdApp.NewSession(user, pass, addr)
	return cs, err
}

func (s *SSO) Logout(w http.ResponseWriter, r *http.Request, newURL string) {
	s.EndSession(w, r)
	http.Redirect(w, r, newURL, http.StatusTemporaryRedirect)
}

// StartSession sets a Crowd session cookie.
func (s *SSO) StartSession(w http.ResponseWriter, cs crowd.Session) {
	ck := http.Cookie{
		Name:    s.CookieConfig.Name,
		Domain:  s.CookieConfig.Domain,
		Secure:  s.CookieConfig.Secure,
		Value:   cs.Token,
		Expires: cs.Expires,
	}
	http.SetCookie(w, &ck)
}

// EndSession invalidates the current Crowd session and cookie
func (s *SSO) EndSession(w http.ResponseWriter, r *http.Request) {
	currentCookie, _ := r.Cookie(s.CookieConfig.Name)
	newCookie := &http.Cookie{
		Name:    s.CookieConfig.Name,
		Domain:  s.CookieConfig.Domain,
		Secure:  s.CookieConfig.Secure,
		MaxAge:  -1,
		Expires: time.Unix(1, 0),
		Value:   "LOGGED-OUT",
	}
	s.CrowdApp.InvalidateSession(currentCookie.Value)

	log.Printf("Got cookie to remove: %+v\n", currentCookie)
	log.Printf("Removal cookie: %+v\n", newCookie)
	http.SetCookie(w, newCookie)
}

// Get User information for the current session (by cookie)
func (s *SSO) GetUser(r *http.Request) (u crowd.User, err error) {
	currentCookie, err := r.Cookie(s.CookieConfig.Name)
	if err == http.ErrNoCookie {
		return u, errors.New("no session cookie")
	}

	userSession, err := s.CrowdApp.GetSession(currentCookie.Value)
	if err != nil {
		return u, errors.New("session not valid")
	}

	return userSession.User, nil
}

func loginPage(w http.ResponseWriter, r *http.Request, s *SSO) bool {
	if r.Method == "GET" { // show login page and bail
		showLoginPage(w, s)
		return false
	} else if r.Method == "POST" {
		user := r.FormValue("username")
		pass := r.FormValue("password")
		host, err := s.ClientAddressFinder(r)
		if err != nil {
			log.Printf("crowdauth: could not get remote addr: %s\n", err)
			showLoginPage(w, s)
			return false
		}

		sess, err := s.Login(user, pass, host)
		if err != nil {
			log.Printf("crowdauth: login/new session failed: %s\n", err)
			showLoginPage(w, s)
			return false
		}

		s.StartSession(w, sess)
	} else {
		return false
	}

	return true
}

func showLoginPage(w http.ResponseWriter, s *SSO) {
	err := s.LoginTemplate.ExecuteTemplate(w, "authPage", nil)
	if err != nil {
		log.Printf("crowdauth: could not exec template: %s\n", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/control-center/serviced/commons/layer"
	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/node"
	"github.com/control-center/serviced/shell"
	"github.com/control-center/serviced/utils"
	"github.com/zenoss/glog"
	dockerclient "github.com/zenoss/go-dockerclient"
)

// ShellConfig is the deserialized object from the command-line
type ShellConfig struct {
	ServiceID        string
	Command          string
	Args             []string
	Username         string
	SaveAs           string
	IsTTY            bool
	Mounts           []string
	ServicedEndpoint string
	LogToStderr      bool
	LogStash         struct {
		Enable        bool
		SettleTime    string
		IdleFlushTime string
	}
}

// getServiceBindMounts retrieves a service's bindmounts
func getServiceBindMounts(lbClientPort string, serviceID string) (map[string]string, error) {
	client, err := node.NewLBClient(lbClientPort)
	if err != nil {
		glog.Errorf("Could not create a client to endpoint: %s, %s", lbClientPort, err)
		return nil, err
	}
	defer client.Close()

	var bindmounts map[string]string
	err = client.GetServiceBindMounts(serviceID, &bindmounts)
	if err != nil {
		if strings.HasPrefix(err.Error(), "rpc: can't find service") {
			glog.Errorf("`serviced service shell` is available only when running serviced in agent mode")
			return nil, err
		}
		glog.Errorf("Error getting service %s's bindmounts, error: %s", serviceID, err)
		return nil, err
	}

	glog.V(1).Infof("getServiceBindMounts: service id=%s: %s", serviceID, bindmounts)
	return bindmounts, nil
}

func buildMounts(lbClientPort string, serviceID string, defaultMounts []string) ([]string, error) {
	bindmounts, err := getServiceBindMounts(lbClientPort, serviceID)
	if err != nil {
		return nil, err
	}

	mounts := defaultMounts
	for hostPath, containerPath := range bindmounts {
		bind := hostPath + "," + containerPath
		mounts = append(mounts, bind)
	}

	return mounts, nil
}

// StartShell runs a command for a given service
func (a *api) StartShell(config ShellConfig) error {
	mounts, err := buildMounts(config.ServicedEndpoint, config.ServiceID, config.Mounts)
	if err != nil {
		return err
	}

	command := append([]string{config.Command}, config.Args...)

	cfg := shell.ProcessConfig{
		ServiceID: config.ServiceID,
		IsTTY:     config.IsTTY,
		SaveAs:    config.SaveAs,
		Mount:     mounts,
		Command:   utils.ShellQuoteArgs(command),
	}

	// TODO: change me to use sockets
	cmd, err := shell.StartDocker(&cfg, options.Endpoint)
	if err != nil {
		return fmt.Errorf("failed to connect to service: %s", err)
	}

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// RunShell runs a predefined service shell command via the service definition
func (a *api) RunShell(config ShellConfig) error {
	client, err := a.connectDAO()
	if err != nil {
		return err
	}

	svc, err := a.GetService(config.ServiceID)
	if err != nil {
		return err
	}

	getSvc := func(svcID string) (service.Service, error) {
		s := service.Service{}
		err := client.GetService(svcID, &s)
		return s, err
	}

	findChild := func(svcID, childName string) (service.Service, error) {
		s := service.Service{}
		err := client.FindChildService(dao.FindChildRequest{svcID, childName}, &s)
		return s, err
	}

	if err := svc.EvaluateRunsTemplate(getSvc, findChild); err != nil {
		fmt.Errorf("error evaluating service:%s Runs:%+v  error:%s", svc.ID, svc.Runs, err)
	}
	command, ok := svc.Runs[config.Command]
	if !ok {
		return fmt.Errorf("command not found for service")
	}
	mounts, err := buildMounts(config.ServicedEndpoint, config.ServiceID, config.Mounts)
	if err != nil {
		return err
	}

	quotedArgs := utils.ShellQuoteArgs(config.Args)
	command = strings.Join([]string{command, quotedArgs}, " ")

	asUser := "su - root -c "
	if config.Username != "" && config.Username != "root" {
		asUser = fmt.Sprintf("su - %s -c ", config.Username)
	}

	cfg := shell.ProcessConfig{
		ServiceID:   config.ServiceID,
		IsTTY:       config.IsTTY,
		SaveAs:      config.SaveAs,
		Mount:       mounts,
		Command:     asUser + utils.ShellQuoteArg(command),
		LogToStderr: config.LogToStderr,
	}

	cfg.LogStash.Enable = config.LogStash.Enable
	cfg.LogStash.SettleTime, err = time.ParseDuration(config.LogStash.SettleTime)
	if err != nil {
		return err
	}

	cfg.LogStash.IdleFlushTime, err = time.ParseDuration(config.LogStash.IdleFlushTime)
	if err != nil {
		return err
	}

	// TODO: change me to use sockets
	cmd, err := shell.StartDocker(&cfg, options.Endpoint)
	if err != nil {
		return fmt.Errorf("failed to connect to service: %s", err)
	}

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if _, ok := utils.GetExitStatus(err); !ok {
		glog.Fatalf("abnormal termination from shell command: %s", err)
	}

	dockercli, err := a.connectDocker()
	if err != nil {
		glog.Fatalf("unable to connect to the docker service: %s", err)
	}
	exitcode, err := dockercli.WaitContainer(config.SaveAs)
	if err != nil {
		glog.Fatalf("failure waiting for container: %s", err)
	}
	container, err := dockercli.InspectContainer(config.SaveAs)
	if err != nil {
		glog.Fatalf("cannot acquire information about container: %s (%s)", config.SaveAs, err)
	}
	glog.V(2).Infof("Container ID: %s", container.ID)

	switch exitcode {
	case 0:
		// Commit the container
		label := ""
		glog.V(0).Infof("Committing container")
		if err := client.Commit(container.ID, &label); err != nil {
			glog.Fatalf("Error committing container: %s (%s)", container.ID, err)
		}
		var layers = 0
		if err := client.ImageLayerCount(container.Image, &layers); err != nil {
			glog.Errorf("Counting layers for image %s", svc.ImageID)
		}
		if layers > layer.WARN_LAYER_COUNT {
			glog.Warningf("Image '%s' number of layers (%d) approaching maximum (%d).  Please squash image layers.",
				svc.ImageID, layers, layer.MAX_LAYER_COUNT)
		}
	default:
		// Delete the container

		if err := dockercli.StopContainer(container.ID, 10); err != nil {
			glog.Fatalf("failed to stop container: %s (%s)", container.ID, err)
		} else if err := dockercli.RemoveContainer(dockerclient.RemoveContainerOptions{ID: container.ID}); err != nil {
			glog.Fatalf("failed to remove container: %s (%s)", container.ID, err)
		}
		return fmt.Errorf("Command returned non-zero exit code %d.  Container not commited.", exitcode)
	}

	return nil
}
<commit_msg>invert precedence of mount sources - cmdline overrides service<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/control-center/serviced/commons/layer"
	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/node"
	"github.com/control-center/serviced/shell"
	"github.com/control-center/serviced/utils"
	"github.com/zenoss/glog"
	dockerclient "github.com/zenoss/go-dockerclient"
)

// ShellConfig is the deserialized object from the command-line
type ShellConfig struct {
	ServiceID        string
	Command          string
	Args             []string
	Username         string
	SaveAs           string
	IsTTY            bool
	Mounts           []string
	ServicedEndpoint string
	LogToStderr      bool
	LogStash         struct {
		Enable        bool
		SettleTime    string
		IdleFlushTime string
	}
}

// getServiceBindMounts retrieves a service's bindmounts
func getServiceBindMounts(lbClientPort string, serviceID string) (map[string]string, error) {
	client, err := node.NewLBClient(lbClientPort)
	if err != nil {
		glog.Errorf("Could not create a client to endpoint: %s, %s", lbClientPort, err)
		return nil, err
	}
	defer client.Close()

	var bindmounts map[string]string
	err = client.GetServiceBindMounts(serviceID, &bindmounts)
	if err != nil {
		if strings.HasPrefix(err.Error(), "rpc: can't find service") {
			glog.Errorf("`serviced service shell` is available only when running serviced in agent mode")
			return nil, err
		}
		glog.Errorf("Error getting service %s's bindmounts, error: %s", serviceID, err)
		return nil, err
	}

	glog.V(1).Infof("getServiceBindMounts: service id=%s: %s", serviceID, bindmounts)
	return bindmounts, nil
}

func buildMounts(lbClientPort string, serviceID string, defaultMounts []string) ([]string, error) {
	bindmounts, err := getServiceBindMounts(lbClientPort, serviceID)
	if err != nil {
		return nil, err
	}

	dsts := map[string]string{}
	for _, mnt := range defaultMounts {
		parts := strings.Split(mnt, ",")
		if len(parts) > 1 {
			dsts[parts[1]] = parts[0] // dsts[dst] = src
		}
	}

	mounts := defaultMounts
	for hostPath, containerPath := range bindmounts {
		if _, ok := dsts[containerPath]; !ok {
			bind := hostPath + "," + containerPath
			mounts = append(mounts, bind)
		}
	}

	return mounts, nil
}

// StartShell runs a command for a given service
func (a *api) StartShell(config ShellConfig) error {
	mounts, err := buildMounts(config.ServicedEndpoint, config.ServiceID, config.Mounts)
	if err != nil {
		return err
	}

	command := append([]string{config.Command}, config.Args...)

	cfg := shell.ProcessConfig{
		ServiceID: config.ServiceID,
		IsTTY:     config.IsTTY,
		SaveAs:    config.SaveAs,
		Mount:     mounts,
		Command:   utils.ShellQuoteArgs(command),
	}

	// TODO: change me to use sockets
	cmd, err := shell.StartDocker(&cfg, options.Endpoint)
	if err != nil {
		return fmt.Errorf("failed to connect to service: %s", err)
	}

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// RunShell runs a predefined service shell command via the service definition
func (a *api) RunShell(config ShellConfig) error {
	client, err := a.connectDAO()
	if err != nil {
		return err
	}

	svc, err := a.GetService(config.ServiceID)
	if err != nil {
		return err
	}

	getSvc := func(svcID string) (service.Service, error) {
		s := service.Service{}
		err := client.GetService(svcID, &s)
		return s, err
	}

	findChild := func(svcID, childName string) (service.Service, error) {
		s := service.Service{}
		err := client.FindChildService(dao.FindChildRequest{svcID, childName}, &s)
		return s, err
	}

	if err := svc.EvaluateRunsTemplate(getSvc, findChild); err != nil {
		fmt.Errorf("error evaluating service:%s Runs:%+v  error:%s", svc.ID, svc.Runs, err)
	}
	command, ok := svc.Runs[config.Command]
	if !ok {
		return fmt.Errorf("command not found for service")
	}
	mounts, err := buildMounts(config.ServicedEndpoint, config.ServiceID, config.Mounts)
	if err != nil {
		return err
	}

	quotedArgs := utils.ShellQuoteArgs(config.Args)
	command = strings.Join([]string{command, quotedArgs}, " ")

	asUser := "su - root -c "
	if config.Username != "" && config.Username != "root" {
		asUser = fmt.Sprintf("su - %s -c ", config.Username)
	}

	cfg := shell.ProcessConfig{
		ServiceID:   config.ServiceID,
		IsTTY:       config.IsTTY,
		SaveAs:      config.SaveAs,
		Mount:       mounts,
		Command:     asUser + utils.ShellQuoteArg(command),
		LogToStderr: config.LogToStderr,
	}

	cfg.LogStash.Enable = config.LogStash.Enable
	cfg.LogStash.SettleTime, err = time.ParseDuration(config.LogStash.SettleTime)
	if err != nil {
		return err
	}

	cfg.LogStash.IdleFlushTime, err = time.ParseDuration(config.LogStash.IdleFlushTime)
	if err != nil {
		return err
	}

	// TODO: change me to use sockets
	cmd, err := shell.StartDocker(&cfg, options.Endpoint)
	if err != nil {
		return fmt.Errorf("failed to connect to service: %s", err)
	}

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if _, ok := utils.GetExitStatus(err); !ok {
		glog.Fatalf("abnormal termination from shell command: %s", err)
	}

	dockercli, err := a.connectDocker()
	if err != nil {
		glog.Fatalf("unable to connect to the docker service: %s", err)
	}
	exitcode, err := dockercli.WaitContainer(config.SaveAs)
	if err != nil {
		glog.Fatalf("failure waiting for container: %s", err)
	}
	container, err := dockercli.InspectContainer(config.SaveAs)
	if err != nil {
		glog.Fatalf("cannot acquire information about container: %s (%s)", config.SaveAs, err)
	}
	glog.V(2).Infof("Container ID: %s", container.ID)

	switch exitcode {
	case 0:
		// Commit the container
		label := ""
		glog.V(0).Infof("Committing container")
		if err := client.Commit(container.ID, &label); err != nil {
			glog.Fatalf("Error committing container: %s (%s)", container.ID, err)
		}
		var layers = 0
		if err := client.ImageLayerCount(container.Image, &layers); err != nil {
			glog.Errorf("Counting layers for image %s", svc.ImageID)
		}
		if layers > layer.WARN_LAYER_COUNT {
			glog.Warningf("Image '%s' number of layers (%d) approaching maximum (%d).  Please squash image layers.",
				svc.ImageID, layers, layer.MAX_LAYER_COUNT)
		}
	default:
		// Delete the container

		if err := dockercli.StopContainer(container.ID, 10); err != nil {
			glog.Fatalf("failed to stop container: %s (%s)", container.ID, err)
		} else if err := dockercli.RemoveContainer(dockerclient.RemoveContainerOptions{ID: container.ID}); err != nil {
			glog.Fatalf("failed to remove container: %s (%s)", container.ID, err)
		}
		return fmt.Errorf("Command returned non-zero exit code %d.  Container not commited.", exitcode)
	}

	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"

	cfg "github.com/tendermint/tendermint/config"
	cmn "github.com/tendermint/tendermint/libs/common"
	"github.com/tendermint/tendermint/p2p"
	"github.com/tendermint/tendermint/privval"
	"github.com/tendermint/tendermint/types"
	tmtime "github.com/tendermint/tendermint/types/time"
)

var (
	nValidators    int
	nNonValidators int
	outputDir      string
	nodeDirPrefix  string

	populatePersistentPeers bool
	hostnamePrefix          string
	startingIPAddress       string
	p2pPort                 int
)

const (
	nodeDirPerm = 0755
)

func init() {
	TestnetFilesCmd.Flags().IntVar(&nValidators, "v", 4,
		"Number of validators to initialize the testnet with")
	TestnetFilesCmd.Flags().IntVar(&nNonValidators, "n", 0,
		"Number of non-validators to initialize the testnet with")
	TestnetFilesCmd.Flags().StringVar(&outputDir, "o", "./mytestnet",
		"Directory to store initialization data for the testnet")
	TestnetFilesCmd.Flags().StringVar(&nodeDirPrefix, "node-dir-prefix", "node",
		"Prefix the directory name for each node with (node results in node0, node1, ...)")

	TestnetFilesCmd.Flags().BoolVar(&populatePersistentPeers, "populate-persistent-peers", true,
		"Update config of each node with the list of persistent peers build using either hostname-prefix or starting-ip-address")
	TestnetFilesCmd.Flags().StringVar(&hostnamePrefix, "hostname-prefix", "node",
		"Hostname prefix (node results in persistent peers list ID0@node0:26656, ID1@node1:26656, ...)")
	TestnetFilesCmd.Flags().StringVar(&startingIPAddress, "starting-ip-address", "",
		"Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:26656, ID1@192.168.0.2:26656, ...)")
	TestnetFilesCmd.Flags().IntVar(&p2pPort, "p2p-port", 26656,
		"P2P Port")
}

// TestnetFilesCmd allows initialisation of files for a Tendermint testnet.
var TestnetFilesCmd = &cobra.Command{
	Use:   "testnet",
	Short: "Initialize files for a Tendermint testnet",
	Long: `testnet will create "v" + "n" number of directories and populate each with
necessary files (private validator, genesis, config, etc.).

Note, strict routability for addresses is turned off in the config file.

Optionally, it will fill in persistent_peers list in config file using either hostnames or IPs.

Example:

	tendermint testnet --v 4 --o ./output --populate-persistent-peers --starting-ip-address 192.168.10.2
	`,
	RunE: testnetFiles,
}

func testnetFiles(cmd *cobra.Command, args []string) error {
	config := cfg.DefaultConfig()
	genVals := make([]types.GenesisValidator, nValidators)

	for i := 0; i < nValidators; i++ {
		nodeDirName := fmt.Sprintf("%s%d", nodeDirPrefix, i)
		nodeDir := filepath.Join(outputDir, nodeDirName)
		config.SetRoot(nodeDir)

		err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm)
		if err != nil {
			_ = os.RemoveAll(outputDir)
			return err
		}

		initFilesWithConfig(config)

		pvFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidator)
		pv := privval.LoadFilePV(pvFile)
		genVals[i] = types.GenesisValidator{
			Address: pv.GetPubKey().Address(),
			PubKey:  pv.GetPubKey(),
			Power:   1,
			Name:    nodeDirName,
		}
	}

	for i := 0; i < nNonValidators; i++ {
		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i+nValidators))
		config.SetRoot(nodeDir)

		err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm)
		if err != nil {
			_ = os.RemoveAll(outputDir)
			return err
		}

		initFilesWithConfig(config)
	}

	// Generate genesis doc from generated validators
	genDoc := &types.GenesisDoc{
		GenesisTime: tmtime.Now(),
		ChainID:     "chain-" + cmn.RandStr(6),
		Validators:  genVals,
	}

	// Write genesis file.
	for i := 0; i < nValidators+nNonValidators; i++ {
		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
		if err := genDoc.SaveAs(filepath.Join(nodeDir, config.BaseConfig.Genesis)); err != nil {
			_ = os.RemoveAll(outputDir)
			return err
		}
	}

	if populatePersistentPeers {
		err := populatePersistentPeersInConfigAndWriteIt(config)
		if err != nil {
			_ = os.RemoveAll(outputDir)
			return err
		}
	}

	fmt.Printf("Successfully initialized %v node directories\n", nValidators+nNonValidators)
	return nil
}

func hostnameOrIP(i int) string {
	if startingIPAddress != "" {
		ip := net.ParseIP(startingIPAddress)
		ip = ip.To4()
		if ip == nil {
			fmt.Printf("%v: non ipv4 address\n", startingIPAddress)
			os.Exit(1)
		}

		for j := 0; j < i; j++ {
			ip[3]++
		}
		return ip.String()
	}

	return fmt.Sprintf("%s%d", hostnamePrefix, i)
}

func populatePersistentPeersInConfigAndWriteIt(config *cfg.Config) error {
	persistentPeers := make([]string, nValidators+nNonValidators)
	for i := 0; i < nValidators+nNonValidators; i++ {
		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
		config.SetRoot(nodeDir)
		nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile())
		if err != nil {
			return err
		}
		persistentPeers[i] = p2p.IDAddressString(nodeKey.ID(), fmt.Sprintf("%s:%d", hostnameOrIP(i), p2pPort))
	}
	persistentPeersList := strings.Join(persistentPeers, ",")

	for i := 0; i < nValidators+nNonValidators; i++ {
		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
		config.SetRoot(nodeDir)
		config.P2P.PersistentPeers = persistentPeersList
		config.P2P.AddrBookStrict = false

		// overwrite default config
		cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config)
	}

	return nil
}
<commit_msg>turn off strict routability every time (#2983)<commit_after>package commands

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"

	cfg "github.com/tendermint/tendermint/config"
	cmn "github.com/tendermint/tendermint/libs/common"
	"github.com/tendermint/tendermint/p2p"
	"github.com/tendermint/tendermint/privval"
	"github.com/tendermint/tendermint/types"
	tmtime "github.com/tendermint/tendermint/types/time"
)

var (
	nValidators    int
	nNonValidators int
	outputDir      string
	nodeDirPrefix  string

	populatePersistentPeers bool
	hostnamePrefix          string
	startingIPAddress       string
	p2pPort                 int
)

const (
	nodeDirPerm = 0755
)

func init() {
	TestnetFilesCmd.Flags().IntVar(&nValidators, "v", 4,
		"Number of validators to initialize the testnet with")
	TestnetFilesCmd.Flags().IntVar(&nNonValidators, "n", 0,
		"Number of non-validators to initialize the testnet with")
	TestnetFilesCmd.Flags().StringVar(&outputDir, "o", "./mytestnet",
		"Directory to store initialization data for the testnet")
	TestnetFilesCmd.Flags().StringVar(&nodeDirPrefix, "node-dir-prefix", "node",
		"Prefix the directory name for each node with (node results in node0, node1, ...)")

	TestnetFilesCmd.Flags().BoolVar(&populatePersistentPeers, "populate-persistent-peers", true,
		"Update config of each node with the list of persistent peers build using either hostname-prefix or starting-ip-address")
	TestnetFilesCmd.Flags().StringVar(&hostnamePrefix, "hostname-prefix", "node",
		"Hostname prefix (node results in persistent peers list ID0@node0:26656, ID1@node1:26656, ...)")
	TestnetFilesCmd.Flags().StringVar(&startingIPAddress, "starting-ip-address", "",
		"Starting IP address (192.168.0.1 results in persistent peers list ID0@192.168.0.1:26656, ID1@192.168.0.2:26656, ...)")
	TestnetFilesCmd.Flags().IntVar(&p2pPort, "p2p-port", 26656,
		"P2P Port")
}

// TestnetFilesCmd allows initialisation of files for a Tendermint testnet.
var TestnetFilesCmd = &cobra.Command{
	Use:   "testnet",
	Short: "Initialize files for a Tendermint testnet",
	Long: `testnet will create "v" + "n" number of directories and populate each with
necessary files (private validator, genesis, config, etc.).

Note, strict routability for addresses is turned off in the config file.

Optionally, it will fill in persistent_peers list in config file using either hostnames or IPs.

Example:

	tendermint testnet --v 4 --o ./output --populate-persistent-peers --starting-ip-address 192.168.10.2
	`,
	RunE: testnetFiles,
}

func testnetFiles(cmd *cobra.Command, args []string) error {
	config := cfg.DefaultConfig()
	genVals := make([]types.GenesisValidator, nValidators)

	for i := 0; i < nValidators; i++ {
		nodeDirName := fmt.Sprintf("%s%d", nodeDirPrefix, i)
		nodeDir := filepath.Join(outputDir, nodeDirName)
		config.SetRoot(nodeDir)

		err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm)
		if err != nil {
			_ = os.RemoveAll(outputDir)
			return err
		}

		initFilesWithConfig(config)

		pvFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidator)
		pv := privval.LoadFilePV(pvFile)
		genVals[i] = types.GenesisValidator{
			Address: pv.GetPubKey().Address(),
			PubKey:  pv.GetPubKey(),
			Power:   1,
			Name:    nodeDirName,
		}
	}

	for i := 0; i < nNonValidators; i++ {
		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i+nValidators))
		config.SetRoot(nodeDir)

		err := os.MkdirAll(filepath.Join(nodeDir, "config"), nodeDirPerm)
		if err != nil {
			_ = os.RemoveAll(outputDir)
			return err
		}

		initFilesWithConfig(config)
	}

	// Generate genesis doc from generated validators
	genDoc := &types.GenesisDoc{
		GenesisTime: tmtime.Now(),
		ChainID:     "chain-" + cmn.RandStr(6),
		Validators:  genVals,
	}

	// Write genesis file.
	for i := 0; i < nValidators+nNonValidators; i++ {
		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
		if err := genDoc.SaveAs(filepath.Join(nodeDir, config.BaseConfig.Genesis)); err != nil {
			_ = os.RemoveAll(outputDir)
			return err
		}
	}

	// Gather persistent peer addresses.
	var (
		persistentPeers string
		err             error
	)
	if populatePersistentPeers {
		persistentPeers, err = persistentPeersString(config)
		if err != nil {
			_ = os.RemoveAll(outputDir)
			return err
		}
	}

	// Overwrite default config.
	for i := 0; i < nValidators+nNonValidators; i++ {
		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
		config.SetRoot(nodeDir)
		config.P2P.AddrBookStrict = false
		if populatePersistentPeers {
			config.P2P.PersistentPeers = persistentPeers
		}

		cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config)
	}

	fmt.Printf("Successfully initialized %v node directories\n", nValidators+nNonValidators)
	return nil
}

func hostnameOrIP(i int) string {
	if startingIPAddress != "" {
		ip := net.ParseIP(startingIPAddress)
		ip = ip.To4()
		if ip == nil {
			fmt.Printf("%v: non ipv4 address\n", startingIPAddress)
			os.Exit(1)
		}

		for j := 0; j < i; j++ {
			ip[3]++
		}
		return ip.String()
	}

	return fmt.Sprintf("%s%d", hostnamePrefix, i)
}

func persistentPeersString(config *cfg.Config) (string, error) {
	persistentPeers := make([]string, nValidators+nNonValidators)
	for i := 0; i < nValidators+nNonValidators; i++ {
		nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i))
		config.SetRoot(nodeDir)
		nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile())
		if err != nil {
			return "", err
		}
		persistentPeers[i] = p2p.IDAddressString(nodeKey.ID(), fmt.Sprintf("%s:%d", hostnameOrIP(i), p2pPort))
	}
	return strings.Join(persistentPeers, ","), nil
}
<|endoftext|>


<commit_before>/*
 * Script that runs srcgraph make -test on a set of test repositories
 */
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/aybabtme/color/brush"
)

type TestCase struct {
	Name     string
	CloneURL string
	CommitID string
}

var tests = []TestCase{
	{"go-sample-0", "https://github.com/sgtest/go-sample-0", "1dd4664fec342c0727850380931429a5850a4402"},
	{"python-sample-0", "https://github.com/sgtest/python-sample-0", "7f4959756bdfc406f318fedfe9f8e8ba98dfe48b"},
	{"python-sample-1", "https://github.com/sgtest/python-sample-1", "0b7a781f8bebf79d36fa8aacb400ca6b892cba95"},
	{"javascript-nodejs-sample-0", "https://github.com/sgtest/javascript-nodejs-sample-0", "e10faf45fd536676a48bbbdb6ab650e7721782bb"},
	{"javascript-nodejs-xrefs-0", "https://github.com/sgtest/javascript-nodejs-xrefs-0", "a82948d15bfcbac86530caf0e9c0929e6c41c353"},
}

var verbose = flag.Bool("verbose", false, "if verbose is true, print individual test output")
var list = flag.Bool("list", false, "list all test cases")
var match = flag.String("match", "", "match cases (by name substring)")

func main() {
	flag.Parse()
	if *list {
		for _, test := range tests {
			fmt.Printf("%s (%s:%s)\n", test.Name, test.CloneURL, test.CommitID)
		}
		os.Exit(0)
	}

	var exitCode = 0
	testDir, err := ioutil.TempDir("", "sg-test")
	if err != nil {
		log.Print(err)
		os.Exit(1)
	}
	defer os.RemoveAll(testDir)

	for _, test := range tests {
		if *match != "" && !strings.Contains(test.Name, *match) {
			continue
		}

		fmt.Printf("Testing %s...\n", test.Name)
		err := testRepo(test, testDir)
		if err != nil {
			exitCode = 1
			fmt.Printf(brush.Red("Test %s failed\n").String(), test.Name)
		} else {
			fmt.Printf(brush.Green("Test %s succeeded\n").String(), test.Name)
		}
	}

	os.Exit(exitCode)
}

func testRepo(t TestCase, testDir string) error {
	{
		cloneCmd := exec.Command("git", "clone", t.CloneURL, t.Name)
		cloneCmd.Dir = testDir
		if *verbose {
			cloneCmd.Stdout, cloneCmd.Stderr = os.Stdout, os.Stderr
		}
		err := cloneCmd.Run()
		if err != nil {
			return err
		}
	}
	{
		checkoutCmd := exec.Command("git", "checkout", t.CommitID)
		checkoutCmd.Dir = filepath.Join(testDir, t.Name)
		if *verbose {
			checkoutCmd.Stdout, checkoutCmd.Stderr = os.Stdout, os.Stderr
		}
		err := checkoutCmd.Run()
		if err != nil {
			return fmt.Errorf("Error (%s)", err)
		}
	}
	{
		srcgraphCmd := exec.Command("srcgraph", "make", "-test")
		srcgraphCmd.Dir = filepath.Join(testDir, t.Name)
		if *verbose {
			srcgraphCmd.Stdout, srcgraphCmd.Stderr = os.Stdout, os.Stderr
		}
		err := srcgraphCmd.Run()
		if err != nil {
			return fmt.Errorf("Error (%s)", err)
		}
	}
	return nil
}
<commit_msg>remove old test_srcgraph<commit_after><|endoftext|>


<commit_before>package ddtxn

import (
	"ddtxn/wfmutex"
	"flag"
	"log"
	"sync"
	"sync/atomic"
)

var Conflicts = flag.Bool("conflicts", false, "Measure conflicts\n")

type KeyType int

const (
	SUM = iota
	MAX
	WRITE
	LIST
	ORDERED_PAIR
)

type BRecord struct {
	padding   [128]byte
	key       Key
	key_type  KeyType
	dd        bool
	last      wfmutex.WFMutex
	value     Value
	int_value int32
	entries   []Entry
	mu        sync.RWMutex
	conflict  int32 // how many times was the lock already held when someone wanted it
	padding1  [128]byte
}

func MakeBR(k Key, val Value, kt KeyType) *BRecord {
	//dlog.Printf("Making %v %v %v\n", k, val, kt)
	b := &BRecord{
		key:      k,
		last:     wfmutex.WFMutex{},
		key_type: kt,
	}
	switch kt {
	case SUM:
		if val != nil {
			b.int_value = val.(int32)
		}
	case MAX:
		if val != nil {
			b.int_value = val.(int32)
		}
	case WRITE:
		if val != nil {
			b.value = val
		}
	case LIST:
		if val == nil {
			b.entries = make([]Entry, 0)
		} else {
			b.entries = make([]Entry, 1)
			b.entries[0] = val.(Entry)
		}
	case ORDERED_PAIR:
		return nil
	}
	return b
}

func (br *BRecord) SLock() {
	br.mu.Lock()
}

func (br *BRecord) SUnlock() {
	br.mu.Unlock()
}

func (br *BRecord) SRLock() {
	br.mu.RLock()
}

func (br *BRecord) SRUnlock() {
	br.mu.RUnlock()
}

func (br *BRecord) Value() Value {
	switch br.key_type {
	case SUM:
		return br.int_value
	case MAX:
		return br.int_value
	case WRITE:
		return br.value
	case LIST:
		return br.entries
	}
	return nil
}

// Used during "normal" phase
func (br *BRecord) Lock() bool {
	x := br.last.Lock()
	if *Conflicts {
		if !x {
			atomic.AddInt32(&br.conflict, 1)
		}
	}
	return x
}

func (br *BRecord) Unlock(tid TID) {
	br.last.Unlock(uint64(tid))
}

func (br *BRecord) IsUnlocked() (bool, uint64) {
	x := br.last.Read()
	if x&wfmutex.LOCKED != 0 {
		if *Conflicts {
			// warning!  turning a read-only thing into a read/write!
			atomic.AddInt32(&br.conflict, 1)
		}
		return false, x
	}
	return true, x
}

func (br *BRecord) Verify(last uint64) bool {
	ok, new_last := br.IsUnlocked()
	if !ok {
		return false
	}
	if uint64(new_last) != last {
		if *Conflicts {
			atomic.AddInt32(&br.conflict, 1)
		}
		return false
	}
	return true
}

func (br *BRecord) Own(last uint64) bool {
	ok, new_last := br.IsUnlocked()
	if ok {
		// Not locked, I can't own it.
		return false
	}
	if uint64(new_last) != wfmutex.LOCKED|last {
		if *Conflicts {
			atomic.AddInt32(&br.conflict, 1)
		}
		return false
	}
	return true
}

// Used during "merge" phase, along with br.mu
func (br *BRecord) Apply(val Value) {
	switch br.key_type {
	case SUM:
		delta := val.(int32)
		atomic.AddInt32(&br.int_value, delta)
	case MAX:
		delta := val.(int32)
		br.mu.Lock()
		defer br.mu.Unlock()
		if br.int_value < delta {
			br.int_value = delta
		}
	case WRITE:
		br.mu.Lock()
		defer br.mu.Unlock()
		br.value = val
	case LIST:
		br.mu.Lock()
		defer br.mu.Unlock()
		entries := val.([]Entry)
		br.listApply(entries)
	}
	// if *Conflicts {
	// 	br.conflict = 0
	// }
}

type Entry struct {
	order int
	key   Key
	top   int
}

const (
	DEFAULT_LIST_SIZE = 10
)

func (br *BRecord) AddOneToRecord(e Entry) {
	br.entries = AddOneToList(br.entries, e)
}

func AddOneToList(lst []Entry, e Entry) []Entry {
	added := false
	for i := 0; i < len(lst); i++ {
		if lst[i].order < e.order {
			lst := append(lst, Entry{})
			copy(lst[i+1:], lst[i:])
			lst[i] = e
			added = true
			break
		}
	}
	if added {
		if len(lst) <= DEFAULT_LIST_SIZE {
			//
		} else {
			lst = lst[:DEFAULT_LIST_SIZE]
		}
	} else if len(lst) < DEFAULT_LIST_SIZE {
		// This goes at the end
		lst = append(lst, e)
	} else {
		lst = lst[:DEFAULT_LIST_SIZE]
	}
	if len(lst) > DEFAULT_LIST_SIZE {
		log.Fatalf("How did this happen?  %v %v %v\n", e, lst, lst[:DEFAULT_LIST_SIZE])
	}
	return lst
}

// default desc
func (br *BRecord) listApply(entries []Entry) {
	lidx := 0
	widx := 0
	i := 0
	new_entries := make([]Entry, DEFAULT_LIST_SIZE)
	for i < DEFAULT_LIST_SIZE {
		if lidx < len(br.entries) && widx < len(entries) && br.entries[lidx].order > entries[widx].order {
			new_entries[i] = br.entries[lidx]
			lidx++
			i++
		} else if widx < len(entries) {
			new_entries[i] = entries[widx]
			widx++
			i++
		} else if lidx < len(br.entries) {
			new_entries[i] = br.entries[lidx]
			widx++
			i++
		} else {
			break
		}
	}
	br.entries = new_entries[:i]
}
<commit_msg>use my own spinlocks<commit_after>package ddtxn

import (
	"ddtxn/spinlock"
	"ddtxn/wfmutex"
	"flag"
	"log"
	"sync"
	"sync/atomic"
)

var Conflicts = flag.Bool("conflicts", false, "Measure conflicts\n")

var Spinlock = flag.Bool("spinlock", true, "Use spinlocks for 2PL\n")

type KeyType int

const (
	SUM = iota
	MAX
	WRITE
	LIST
	ORDERED_PAIR
)

type BRecord struct {
	padding   [128]byte
	key       Key
	key_type  KeyType
	dd        bool
	last      wfmutex.WFMutex
	lock      spinlock.RWSpinlock
	value     Value
	int_value int32
	entries   []Entry
	mu        sync.RWMutex
	conflict  int32 // how many times was the lock already held when someone wanted it
	padding1  [128]byte
}

func MakeBR(k Key, val Value, kt KeyType) *BRecord {
	//dlog.Printf("Making %v %v %v\n", k, val, kt)
	b := &BRecord{
		key:      k,
		last:     wfmutex.WFMutex{},
		key_type: kt,
	}
	switch kt {
	case SUM:
		if val != nil {
			b.int_value = val.(int32)
		}
	case MAX:
		if val != nil {
			b.int_value = val.(int32)
		}
	case WRITE:
		if val != nil {
			b.value = val
		}
	case LIST:
		if val == nil {
			b.entries = make([]Entry, 0)
		} else {
			b.entries = make([]Entry, 1)
			b.entries[0] = val.(Entry)
		}
	case ORDERED_PAIR:
		return nil
	}
	return b
}

func (br *BRecord) SLock() {
	if *Spinlock {
		br.lock.Lock()
	} else {
		br.mu.Lock()
	}
}

func (br *BRecord) SUnlock() {
	if *Spinlock {
		br.lock.Unlock()
	} else {
		br.mu.Unlock()
	}
}

func (br *BRecord) SRLock() {
	if *Spinlock {
		br.lock.RLock()
	} else {
		br.mu.RLock()
	}
}

func (br *BRecord) SRUnlock() {
	if *Spinlock {
		br.lock.RUnlock()
	} else {
		br.mu.RUnlock()
	}
}

func (br *BRecord) Value() Value {
	switch br.key_type {
	case SUM:
		return br.int_value
	case MAX:
		return br.int_value
	case WRITE:
		return br.value
	case LIST:
		return br.entries
	}
	return nil
}

// Used during "normal" phase
func (br *BRecord) Lock() bool {
	x := br.last.Lock()
	if *Conflicts {
		if !x {
			atomic.AddInt32(&br.conflict, 1)
		}
	}
	return x
}

func (br *BRecord) Unlock(tid TID) {
	br.last.Unlock(uint64(tid))
}

func (br *BRecord) IsUnlocked() (bool, uint64) {
	x := br.last.Read()
	if x&wfmutex.LOCKED != 0 {
		if *Conflicts {
			// warning!  turning a read-only thing into a read/write!
			atomic.AddInt32(&br.conflict, 1)
		}
		return false, x
	}
	return true, x
}

func (br *BRecord) Verify(last uint64) bool {
	ok, new_last := br.IsUnlocked()
	if !ok {
		return false
	}
	if uint64(new_last) != last {
		if *Conflicts {
			atomic.AddInt32(&br.conflict, 1)
		}
		return false
	}
	return true
}

func (br *BRecord) Own(last uint64) bool {
	ok, new_last := br.IsUnlocked()
	if ok {
		// Not locked, I can't own it.
		return false
	}
	if uint64(new_last) != wfmutex.LOCKED|last {
		if *Conflicts {
			atomic.AddInt32(&br.conflict, 1)
		}
		return false
	}
	return true
}

// Used during "merge" phase, along with br.mu
func (br *BRecord) Apply(val Value) {
	switch br.key_type {
	case SUM:
		delta := val.(int32)
		atomic.AddInt32(&br.int_value, delta)
	case MAX:
		delta := val.(int32)
		br.mu.Lock()
		defer br.mu.Unlock()
		if br.int_value < delta {
			br.int_value = delta
		}
	case WRITE:
		br.mu.Lock()
		defer br.mu.Unlock()
		br.value = val
	case LIST:
		br.mu.Lock()
		defer br.mu.Unlock()
		entries := val.([]Entry)
		br.listApply(entries)
	}
	// if *Conflicts {
	// 	br.conflict = 0
	// }
}

type Entry struct {
	order int
	key   Key
	top   int
}

const (
	DEFAULT_LIST_SIZE = 10
)

func (br *BRecord) AddOneToRecord(e Entry) {
	br.entries = AddOneToList(br.entries, e)
}

func AddOneToList(lst []Entry, e Entry) []Entry {
	added := false
	for i := 0; i < len(lst); i++ {
		if lst[i].order < e.order {
			lst := append(lst, Entry{})
			copy(lst[i+1:], lst[i:])
			lst[i] = e
			added = true
			break
		}
	}
	if added {
		if len(lst) <= DEFAULT_LIST_SIZE {
			//
		} else {
			lst = lst[:DEFAULT_LIST_SIZE]
		}
	} else if len(lst) < DEFAULT_LIST_SIZE {
		// This goes at the end
		lst = append(lst, e)
	} else {
		lst = lst[:DEFAULT_LIST_SIZE]
	}
	if len(lst) > DEFAULT_LIST_SIZE {
		log.Fatalf("How did this happen?  %v %v %v\n", e, lst, lst[:DEFAULT_LIST_SIZE])
	}
	return lst
}

// default desc
func (br *BRecord) listApply(entries []Entry) {
	lidx := 0
	widx := 0
	i := 0
	new_entries := make([]Entry, DEFAULT_LIST_SIZE)
	for i < DEFAULT_LIST_SIZE {
		if lidx < len(br.entries) && widx < len(entries) && br.entries[lidx].order > entries[widx].order {
			new_entries[i] = br.entries[lidx]
			lidx++
			i++
		} else if widx < len(entries) {
			new_entries[i] = entries[widx]
			widx++
			i++
		} else if lidx < len(br.entries) {
			new_entries[i] = br.entries[lidx]
			widx++
			i++
		} else {
			break
		}
	}
	br.entries = new_entries[:i]
}
<|endoftext|>


<commit_before>package navitia

import (
	"context"
	"net/url"
	"strconv"

	"github.com/aabizri/navitia/types"
)

const regionEndpoint string = "coverage"

// A RegionResults holds results for a coverage query
//
// This Results doesn't support paging :(
type RegionResults struct {
	// The list of regions retrieved
	Regions []types.Region `json:"Regions"`

	// Timing information
	Logging

	// Held session
	session *Session
}

// RegionRequest contains the parameters needed to make a Coverage request
type RegionRequest struct {
	// Count is the number of items to return, if count=0, then it will return the default number
	// BUG: Count doesn't work, server-side.
	Count uint

	// Disable GeoJSON for the reply. GeoJSON object can be VERY large ! >1MB;
	DisableGeoJSON bool
}

func (req RegionRequest) toURL() (url.Values, error) {
	params := url.Values{}

	if count := req.Count; count != 0 {
		countStr := strconv.FormatUint(uint64(count), 10)
		params["count"] = []string{countStr}
	}

	return params, nil
}

func (s *Session) region(ctx context.Context, url string, params RegionRequest) (*RegionResults, error) {
	var results = &RegionResults{session: s}
	err := s.request(ctx, url, params, results)
	return results, err
}

// Regions lists the areas covered by the Navitia API.
// i.e it returns the coverage of the API.
//
// It is context aware.
func (s *Session) Regions(ctx context.Context, req RegionRequest) (*RegionResults, error) {
	// Create the URL
	url := s.APIURL + "/" + regionEndpoint

	// Call and return
	return s.region(ctx, url, req)
}

// RegionByID provides information about a specific region.
// If the ID provided isn't an ID of a region, this WILL fail.
//
// It is context aware.
func (s *Session) RegionByID(ctx context.Context, req RegionRequest, id types.ID) (*RegionResults, error) {
	// Build the URL
	url := s.APIURL + "/" + regionEndpoint + "/" + string(id)

	// Call and return
	return s.region(ctx, url, req)
}

// RegionByPos provides information about the region englobing the specific position.
//
// It is context aware.
func (s *Session) RegionByPos(ctx context.Context, req RegionRequest, coords types.Coordinates) (*RegionResults, error) {
	// Build the URL
	coordsFormatted := coords.QueryEscape()
	url := s.APIURL + "/" + regionEndpoint + "/" + coordsFormatted

	// Call and return
	return s.region(ctx, url, req)
}
<commit_msg>Stylistic fix for RegionByID comment<commit_after>package navitia

import (
	"context"
	"net/url"
	"strconv"

	"github.com/aabizri/navitia/types"
)

const regionEndpoint string = "coverage"

// A RegionResults holds results for a coverage query
//
// This Results doesn't support paging :(
type RegionResults struct {
	// The list of regions retrieved
	Regions []types.Region `json:"Regions"`

	// Timing information
	Logging

	// Held session
	session *Session
}

// RegionRequest contains the parameters needed to make a Coverage request
type RegionRequest struct {
	// Count is the number of items to return, if count=0, then it will return the default number
	// BUG: Count doesn't work, server-side.
	Count uint

	// Disable GeoJSON for the reply. GeoJSON object can be VERY large ! >1MB;
	DisableGeoJSON bool
}

func (req RegionRequest) toURL() (url.Values, error) {
	params := url.Values{}

	if count := req.Count; count != 0 {
		countStr := strconv.FormatUint(uint64(count), 10)
		params["count"] = []string{countStr}
	}

	return params, nil
}

func (s *Session) region(ctx context.Context, url string, params RegionRequest) (*RegionResults, error) {
	var results = &RegionResults{session: s}
	err := s.request(ctx, url, params, results)
	return results, err
}

// Regions lists the areas covered by the Navitia API.
// i.e it returns the coverage of the API.
//
// It is context aware.
func (s *Session) Regions(ctx context.Context, req RegionRequest) (*RegionResults, error) {
	// Create the URL
	url := s.APIURL + "/" + regionEndpoint

	// Call and return
	return s.region(ctx, url, req)
}

// RegionByID provides information about a specific region.
//
// If the ID provided isn't an ID of a region, this WILL fail.
//
// It is context aware.
func (s *Session) RegionByID(ctx context.Context, req RegionRequest, id types.ID) (*RegionResults, error) {
	// Build the URL
	url := s.APIURL + "/" + regionEndpoint + "/" + string(id)

	// Call and return
	return s.region(ctx, url, req)
}

// RegionByPos provides information about the region englobing the specific position.
//
// It is context aware.
func (s *Session) RegionByPos(ctx context.Context, req RegionRequest, coords types.Coordinates) (*RegionResults, error) {
	// Build the URL
	coordsFormatted := coords.QueryEscape()
	url := s.APIURL + "/" + regionEndpoint + "/" + coordsFormatted

	// Call and return
	return s.region(ctx, url, req)
}
<|endoftext|>