<commit_before>package main

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/rfjakob/gocryptfs/internal/configfile"
	"github.com/rfjakob/gocryptfs/internal/nametransform"
	"github.com/rfjakob/gocryptfs/internal/readpassword"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// initDir prepares a directory for use as a gocryptfs storage directory.
// In forward mode, this means creating the gocryptfs.conf and gocryptfs.diriv
// files in an empty directory.
// In reverse mode, we create .gocryptfs.reverse.conf and the directory does
// not to be empty.
func initDir(args *argContainer) {
	var err error
	if args.reverse {
		_, err = os.Stat(args.config)
		if err == nil {
			tlog.Fatal.Printf("Config file %q already exists", args.config)
			os.Exit(ErrExitInit)
		}
	} else {
		err = checkDirEmpty(args.cipherdir)
		if err != nil {
			tlog.Fatal.Printf("Invalid cipherdir: %v", err)
			os.Exit(ErrExitInit)
		}
	}
	// Choose password for config file
	if args.extpass == "" {
		tlog.Info.Printf("Choose a password for protecting your files.")
	}
	password := readpassword.Twice(args.extpass)
	creator := tlog.ProgramName + " " + GitVersion
	err = configfile.CreateConfFile(args.config, password, args.plaintextnames, args.scryptn, creator, args.aessiv)
	if err != nil {
		tlog.Fatal.Println(err)
		os.Exit(ErrExitInit)
	}
	// Forward mode with filename encryption enabled needs a gocryptfs.diriv
	// in the root dir
	if !args.plaintextnames && !args.reverse {
		err = nametransform.WriteDirIV(args.cipherdir)
		if err != nil {
			tlog.Fatal.Println(err)
			os.Exit(ErrExitInit)
		}
	}
	mountArgs := ""
	fsName := "gocryptfs"
	if args.reverse {
		mountArgs = " -reverse"
		fsName = "gocryptfs-reverse"
	}
	tlog.Info.Printf(tlog.ColorGreen+"The %s filesystem has been created successfully."+tlog.ColorReset,
		fsName)
	wd, _ := os.Getwd()
	friendlyPath, _ := filepath.Rel(wd, args.cipherdir)
	if strings.HasPrefix(friendlyPath, "../") {
		// A relative path that starts with "../" is pretty unfriendly, just
		// keep the absolute path.
		friendlyPath = args.cipherdir
	}
	tlog.Info.Printf(tlog.ColorGrey+"You can now mount it using: %s%s %s MOUNTPOINT"+tlog.ColorReset,
		tlog.ProgramName, mountArgs, friendlyPath)
	os.Exit(0)
}
<commit_msg>main: init: handle spaces in mount suggestion message<commit_after>package main

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/rfjakob/gocryptfs/internal/configfile"
	"github.com/rfjakob/gocryptfs/internal/nametransform"
	"github.com/rfjakob/gocryptfs/internal/readpassword"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// initDir prepares a directory for use as a gocryptfs storage directory.
// In forward mode, this means creating the gocryptfs.conf and gocryptfs.diriv
// files in an empty directory.
// In reverse mode, we create .gocryptfs.reverse.conf and the directory does
// not to be empty.
func initDir(args *argContainer) {
	var err error
	if args.reverse {
		_, err = os.Stat(args.config)
		if err == nil {
			tlog.Fatal.Printf("Config file %q already exists", args.config)
			os.Exit(ErrExitInit)
		}
	} else {
		err = checkDirEmpty(args.cipherdir)
		if err != nil {
			tlog.Fatal.Printf("Invalid cipherdir: %v", err)
			os.Exit(ErrExitInit)
		}
	}
	// Choose password for config file
	if args.extpass == "" {
		tlog.Info.Printf("Choose a password for protecting your files.")
	}
	password := readpassword.Twice(args.extpass)
	creator := tlog.ProgramName + " " + GitVersion
	err = configfile.CreateConfFile(args.config, password, args.plaintextnames, args.scryptn, creator, args.aessiv)
	if err != nil {
		tlog.Fatal.Println(err)
		os.Exit(ErrExitInit)
	}
	// Forward mode with filename encryption enabled needs a gocryptfs.diriv
	// in the root dir
	if !args.plaintextnames && !args.reverse {
		err = nametransform.WriteDirIV(args.cipherdir)
		if err != nil {
			tlog.Fatal.Println(err)
			os.Exit(ErrExitInit)
		}
	}
	mountArgs := ""
	fsName := "gocryptfs"
	if args.reverse {
		mountArgs = " -reverse"
		fsName = "gocryptfs-reverse"
	}
	tlog.Info.Printf(tlog.ColorGreen+"The %s filesystem has been created successfully."+tlog.ColorReset,
		fsName)
	wd, _ := os.Getwd()
	friendlyPath, _ := filepath.Rel(wd, args.cipherdir)
	if strings.HasPrefix(friendlyPath, "../") {
		// A relative path that starts with "../" is pretty unfriendly, just
		// keep the absolute path.
		friendlyPath = args.cipherdir
	}
	if strings.Contains(friendlyPath, " ") {
		friendlyPath = "\"" + friendlyPath + "\""
	}
	tlog.Info.Printf(tlog.ColorGrey+"You can now mount it using: %s%s %s MOUNTPOINT"+tlog.ColorReset,
		tlog.ProgramName, mountArgs, friendlyPath)
	os.Exit(0)
}
<|endoftext|>


<commit_before>package oauth2

import (
	"net/http"
	"net/http/httptest"
	"testing"

	. "github.com/stretchr/testify/assert"
)

var gitlabTestUserResponse = `{
	"id": 1,
	"username": "john_smith",
	"email": "john@example.com",
	"name": "John Smith",
	"state": "active",
	"avatar_url": "http://localhost:3000/uploads/user/avatar/1/index.jpg",
	"web_url": "http://localhost:3000/john_smith",
	"created_at": "2012-05-23T08:00:58Z",
	"bio": null,
	"location": null,
	"public_email": "john@example.com",
	"skype": "",
	"linkedin": "",
	"twitter": "",
	"website_url": "",
	"organization": "",
	"last_sign_in_at": "2012-06-01T11:41:01Z",
	"confirmed_at": "2012-05-23T09:05:22Z",
	"theme_id": 1,
	"last_activity_on": "2012-05-23",
	"color_scheme_id": 2,
	"projects_limit": 100,
	"current_sign_in_at": "2012-06-02T06:36:55Z",
	"identities": [
	  {"provider": "github", "extern_uid": "2435223452345"},
	  {"provider": "bitbucket", "extern_uid": "john_smith"},
	  {"provider": "google_oauth2", "extern_uid": "8776128412476123468721346"}
	],
	"can_create_group": true,
	"can_create_project": true,
	"two_factor_enabled": true,
	"external": false,
	"private_profile": false
  }`

var gitlabTestGroupsResponse = `[
	{
	  "id": 1,
	  "web_url": "https://gitlab.com/groups/example",
	  "name": "example",
	  "path": "example",
	  "description": "",
	  "visibility": "private",
	  "lfs_enabled": true,
	  "avatar_url": null,
	  "request_access_enabled": true,
	  "full_name": "example",
	  "full_path": "example",
	  "parent_id": null,
	  "ldap_cn": null,
	  "ldap_access": null
	},
	{
		"id": 2,
		"web_url": "https://gitlab.com/groups/example/subgroup",
		"name": "subgroup",
		"path": "subgroup",
		"description": "",
		"visibility": "private",
		"lfs_enabled": true,
		"avatar_url": null,
		"request_access_enabled": true,
		"full_name": "example / subgroup",
		"full_path": "example/subgroup",
		"parent_id": null,
		"ldap_cn": null,
		"ldap_access": null
	}
]`

func Test_Gitlab_getUserInfo(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/user" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestUserResponse))
		} else if r.URL.Path == "/groups" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestGroupsResponse))
		}
	}))
	defer server.Close()

	gitlabAPI = server.URL

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	NoError(t, err)
	Equal(t, "john_smith", u.Sub)
	Equal(t, "john@example.com", u.Email)
	Equal(t, "John Smith", u.Name)
	Equal(t, []string{"example", "example/subgroup"}, u.Groups)
	Equal(t, `{"user":`+gitlabTestUserResponse+`,"groups":`+gitlabTestGroupsResponse+`}`, rawJSON)
}
<commit_msg>fixing test coverage<commit_after>package oauth2

import (
	"net/http"
	"net/http/httptest"
	"regexp"
	"testing"

	. "github.com/stretchr/testify/assert"
	"github.com/tarent/loginsrv/model"
)

var gitlabTestUserResponse = `{
	"id": 1,
	"username": "john_smith",
	"email": "john@example.com",
	"name": "John Smith",
	"state": "active",
	"avatar_url": "http://localhost:3000/uploads/user/avatar/1/index.jpg",
	"web_url": "http://localhost:3000/john_smith",
	"created_at": "2012-05-23T08:00:58Z",
	"bio": null,
	"location": null,
	"public_email": "john@example.com",
	"skype": "",
	"linkedin": "",
	"twitter": "",
	"website_url": "",
	"organization": "",
	"last_sign_in_at": "2012-06-01T11:41:01Z",
	"confirmed_at": "2012-05-23T09:05:22Z",
	"theme_id": 1,
	"last_activity_on": "2012-05-23",
	"color_scheme_id": 2,
	"projects_limit": 100,
	"current_sign_in_at": "2012-06-02T06:36:55Z",
	"identities": [
	  {"provider": "github", "extern_uid": "2435223452345"},
	  {"provider": "bitbucket", "extern_uid": "john_smith"},
	  {"provider": "google_oauth2", "extern_uid": "8776128412476123468721346"}
	],
	"can_create_group": true,
	"can_create_project": true,
	"two_factor_enabled": true,
	"external": false,
	"private_profile": false
  }`

var gitlabTestGroupsResponse = `[
	{
	  "id": 1,
	  "web_url": "https://gitlab.com/groups/example",
	  "name": "example",
	  "path": "example",
	  "description": "",
	  "visibility": "private",
	  "lfs_enabled": true,
	  "avatar_url": null,
	  "request_access_enabled": true,
	  "full_name": "example",
	  "full_path": "example",
	  "parent_id": null,
	  "ldap_cn": null,
	  "ldap_access": null
	},
	{
		"id": 2,
		"web_url": "https://gitlab.com/groups/example/subgroup",
		"name": "subgroup",
		"path": "subgroup",
		"description": "",
		"visibility": "private",
		"lfs_enabled": true,
		"avatar_url": null,
		"request_access_enabled": true,
		"full_name": "example / subgroup",
		"full_path": "example/subgroup",
		"parent_id": null,
		"ldap_cn": null,
		"ldap_access": null
	}
]`

func Test_Gitlab_getUserInfo(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/user" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestUserResponse))
		} else if r.URL.Path == "/groups" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestGroupsResponse))
		}
	}))
	defer server.Close()

	gitlabAPI = server.URL

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	NoError(t, err)
	Equal(t, "john_smith", u.Sub)
	Equal(t, "john@example.com", u.Email)
	Equal(t, "John Smith", u.Name)
	Equal(t, []string{"example", "example/subgroup"}, u.Groups)
	Equal(t, `{"user":`+gitlabTestUserResponse+`,"groups":`+gitlabTestGroupsResponse+`}`, rawJSON)
}

func Test_Gitlab_getUserInfo_NoServer(t *testing.T) {
	gitlabAPI = "http://localhost"

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	Equal(t, model.UserInfo{}, u)
	Empty(t, rawJSON)
	Error(t, err)
	Regexp(t, regexp.MustCompile(`connection refused$`), err.Error())
}

func Test_Gitlab_getUserInfo_UserContentTypeNegative(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/user" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.Write([]byte(gitlabTestUserResponse))
		} else if r.URL.Path == "/groups" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestGroupsResponse))
		}
	}))
	defer server.Close()

	gitlabAPI = server.URL

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	Equal(t, model.UserInfo{}, u)
	Empty(t, rawJSON)
	Error(t, err)
	Regexp(t, regexp.MustCompile(`^wrong content-type on gitlab get user info`), err.Error())
}

func Test_Gitlab_getUserInfo_GroupsContentTypeNegative(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/user" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestUserResponse))
		} else if r.URL.Path == "/groups" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.Write([]byte(gitlabTestGroupsResponse))
		}
	}))
	defer server.Close()

	gitlabAPI = server.URL

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	Equal(t, model.UserInfo{}, u)
	Empty(t, rawJSON)
	Error(t, err)
	Regexp(t, regexp.MustCompile(`^wrong content-type on gitlab get groups info`), err.Error())
}

func Test_Gitlab_getUserInfo_UserStatusCodeNegative(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/user" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(gitlabTestUserResponse))
		} else if r.URL.Path == "/groups" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestGroupsResponse))
		}
	}))
	defer server.Close()

	gitlabAPI = server.URL

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	Equal(t, model.UserInfo{}, u)
	Empty(t, rawJSON)
	Error(t, err)
	Regexp(t, regexp.MustCompile(`^got http status [0-9]{3} on gitlab get user info`), err.Error())
}

func Test_Gitlab_getUserInfo_GroupsStatusCodeNegative(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/user" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestUserResponse))
		} else if r.URL.Path == "/groups" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(gitlabTestGroupsResponse))
		}
	}))
	defer server.Close()

	gitlabAPI = server.URL

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	Equal(t, model.UserInfo{}, u)
	Empty(t, rawJSON)
	Error(t, err)
	Regexp(t, regexp.MustCompile(`^got http status [0-9]{3} on gitlab get groups info`), err.Error())
}

func Test_Gitlab_getUserInfo_UserJSONNegative(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/user" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte("[]"))
		} else if r.URL.Path == "/groups" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestGroupsResponse))
		}
	}))
	defer server.Close()

	gitlabAPI = server.URL

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	Equal(t, model.UserInfo{}, u)
	Empty(t, rawJSON)
	Error(t, err)
	Regexp(t, regexp.MustCompile(`^error parsing gitlab get user info`), err.Error())
}

func Test_Gitlab_getUserInfo_GroupsJSONNegative(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/user" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte(gitlabTestUserResponse))
		} else if r.URL.Path == "/groups" {
			Equal(t, "secret", r.FormValue("access_token"))
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			w.Write([]byte("{}"))
		}
	}))
	defer server.Close()

	gitlabAPI = server.URL

	u, rawJSON, err := providerGitlab.GetUserInfo(TokenInfo{AccessToken: "secret"})
	Equal(t, model.UserInfo{}, u)
	Empty(t, rawJSON)
	Error(t, err)
	Regexp(t, regexp.MustCompile(`^error parsing gitlab get groups info`), err.Error())
}
<|endoftext|>


<commit_before>package rpc

import (
	"bufio"
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"encoding/binary"
	"errors"
	"fmt"
	hdfs "github.com/colinmarc/hdfs/protocol/hadoop_hdfs"
	"hash/crc32"
	"io"
	"io/ioutil"
	"math"
	"net"
)

const (
	dataTransferVersion = 0x1c
	readBlockOp         = 0x51
)

// TODO: datanode blacklisting

// BlockReader implements io.ReaderCloser for reading a single block from HDFS,
// abstracting over reading from multiple datanodes.
type BlockReader struct {
	closed bool
	conn   net.Conn
	reader *bufio.Reader

	block       *hdfs.LocatedBlockProto
	checksumTab *crc32.Table

	offset    uint64
	chunkSize uint32
	packet    openPacket
	buf       bytes.Buffer
}

type openPacket struct {
	numChunks     int
	nextChunk     int
	checksumBytes []byte
	blockOffset   uint64
	packetOffset  uint64
	length        uint64
	last          bool
}

// NewBlockReader returns a new BlockReader, given the block information and
// security token from the namenode.
func NewBlockReader(block *hdfs.LocatedBlockProto, offset uint64) (*BlockReader, error) {
	br := &BlockReader{
		block:  block,
		offset: offset,
	}

	// TODO check multiple datanodes
	datanode := br.block.GetLocs()[0].GetId()
	address := fmt.Sprintf("%s:%d", datanode.GetIpAddr(), datanode.GetXferPort())
	err := br.connect(address)
	if err != nil {
		return nil, err
	}

	return br, nil
}

func (br *BlockReader) connect(datanode string) error {
	conn, err := net.DialTimeout("tcp", datanode, connectionTimeout)
	if err != nil {
		return err
	}

	br.conn = conn
	err = br.writeBlockReadRequest()
	if err != nil {
		return err
	}

	br.reader = bufio.NewReader(br.conn)
	resp, err := br.readBlockReadResponse()
	if err != nil {
		return err
	}

	checksumInfo := resp.GetReadOpChecksumInfo().GetChecksum()
	checksumType := checksumInfo.GetType()
	if checksumType == hdfs.ChecksumTypeProto_CHECKSUM_CRC32 {
		br.checksumTab = crc32.IEEETable
	} else if checksumType == hdfs.ChecksumTypeProto_CHECKSUM_CRC32C {
		br.checksumTab = crc32.MakeTable(crc32.Castagnoli)
	} else {
		return fmt.Errorf("Unsupported checksum type:", checksumType)
	}

	br.chunkSize = checksumInfo.GetBytesPerChecksum()
	br.startNewPacket()

	// The read will start aligned to a chunk boundary, so we need to seek forward
	// to the requested offset.
	amountToDiscard := br.offset - br.packet.blockOffset
	if amountToDiscard > 0 {
		io.CopyN(ioutil.Discard, br, int64(amountToDiscard))
	}

	return nil
}

func (br *BlockReader) Close() {
	br.conn.Close()
	br.closed = true
}

func (br *BlockReader) Read(b []byte) (int, error) {
	if br.closed {
		return 0, errors.New("The BlockReader is closed.")
	} else if br.offset >= br.block.GetB().GetNumBytes() {
		br.Close()
		return 0, io.EOF
	}

	// first, read any leftover data from buf
	if br.buf.Len() > 0 {
		n, _ := br.buf.Read(b)
		return n, nil
	}

	if br.packet.nextChunk >= br.packet.numChunks {
		br.startNewPacket()
	}

	// then, read until we fill up b or we reach the end of the packet
	readOffset := 0
	for br.packet.nextChunk < br.packet.numChunks {
		chOff := 4 * br.packet.nextChunk
		checksum := br.packet.checksumBytes[chOff : chOff+4]

		remaining := br.packet.length - br.packet.packetOffset
		chunkLength := int64(math.Min(float64(br.chunkSize), float64(remaining)))

		chunkReader := io.LimitReader(br.reader, int64(chunkLength))
		chunkBytes := b[readOffset:]
		n, err := chunkReader.Read(chunkBytes)

		readOffset += n
		br.packet.packetOffset += uint64(n)
		br.packet.nextChunk++

		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}

			br.Close()
			return readOffset, err
		}

		crc := crc32.Checksum(chunkBytes[:n], br.checksumTab)

		if int64(n) < chunkLength {
			// save any leftovers
			br.buf.Reset()
			leftover, err := br.buf.ReadFrom(chunkReader)
			if err != nil {
				return readOffset, err
			}

			br.packet.packetOffset += uint64(leftover)

			// update the checksum with the leftovers
			crc = crc32.Update(crc, br.checksumTab, br.buf.Bytes())
		}

		if crc != binary.BigEndian.Uint32(checksum) {
			return readOffset, errors.New("Invalid checksum from the datanode!")
		}

		if readOffset == len(b) {
			break
		}
	}

	return readOffset, nil
}

// A read request to a datanode:
// +-----------------------------------------------------------+
// |  Data Transfer Protocol Version, int16                    |
// +-----------------------------------------------------------+
// |  Op code, 1 byte (READ_BLOCK = 0x51)                      |
// +-----------------------------------------------------------+
// |  varint length + OpReadBlockProto                         |
// +-----------------------------------------------------------+
func (br *BlockReader) writeBlockReadRequest() error {
	header := []byte{0x00, dataTransferVersion, readBlockOp}

	// TODO offset/length?
	needed := (br.block.GetB().GetNumBytes() - br.offset)
	op := newReadBlockOp(br.block, br.offset, needed)
	opBytes, err := makeDelimitedMsg(op)
	if err != nil {
		return err
	}

	req := append(header, opBytes...)
	_, err = br.conn.Write(req)
	if err != nil {
		return err
	}

	return nil
}

// The initial response from the datanode:
// +-----------------------------------------------------------+
// |  varint length + BlockOpResponseProto                     |
// +-----------------------------------------------------------+
func (br *BlockReader) readBlockReadResponse() (*hdfs.BlockOpResponseProto, error) {
	respLength, err := binary.ReadUvarint(br.reader)
	if err != nil {
		return nil, err
	}

	respBytes := make([]byte, respLength)
	_, err = io.ReadFull(br.reader, respBytes)
	if err != nil {
		return nil, err
	}

	resp := &hdfs.BlockOpResponseProto{}
	err = proto.Unmarshal(respBytes, resp)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

// A packet from the datanode:
// +-----------------------------------------------------------+
// |  uint32 length of the packet                              |
// +-----------------------------------------------------------+
// |  size of the PacketHeaderProto, uint16                    |
// +-----------------------------------------------------------+
// |  PacketHeaderProto                                        |
// +-----------------------------------------------------------+
// |  N checksums, 4 bytes each                                |
// +-----------------------------------------------------------+
// |  N chunks of payload data                                 |
// +-----------------------------------------------------------+
func (br *BlockReader) startNewPacket() error {
	header, err := br.readPacketHeader()
	if err != nil {
		return err
	}

	blockOffset := uint64(header.GetOffsetInBlock())
	dataLength := uint64(header.GetDataLen())
	numChunks := int(math.Ceil(float64(dataLength) / float64(br.chunkSize)))

	// TODO don't assume checksum size is 4
	br.packet = openPacket{
		numChunks:     numChunks,
		nextChunk:     0,
		checksumBytes: make([]byte, numChunks*4),
		blockOffset:   blockOffset,
		packetOffset:  0,
		length:        dataLength,
		last:          header.GetLastPacketInBlock(),
	}

	_, err = io.ReadFull(br.reader, br.packet.checksumBytes)
	if err != nil {
		return err
	}

	return nil
}

func (br *BlockReader) readPacketHeader() (*hdfs.PacketHeaderProto, error) {
	var packetLength uint32
	err := binary.Read(br.reader, binary.BigEndian, &packetLength)
	if err != nil {
		return nil, err
	}

	var packetHeaderLength uint16
	err = binary.Read(br.reader, binary.BigEndian, &packetHeaderLength)
	if err != nil {
		return nil, err
	}

	packetHeaderBytes := make([]byte, packetHeaderLength)
	_, err = io.ReadFull(br.reader, packetHeaderBytes)
	if err != nil {
		return nil, err
	}

	packetHeader := &hdfs.PacketHeaderProto{}
	err = proto.Unmarshal(packetHeaderBytes, packetHeader)

	return packetHeader, nil
}

func newReadBlockOp(block *hdfs.LocatedBlockProto, offset, length uint64) *hdfs.OpReadBlockProto {
	return &hdfs.OpReadBlockProto{
		Header: &hdfs.ClientOperationHeaderProto{
			BaseHeader: &hdfs.BaseHeaderProto{
				Block: block.GetB(),
				Token: block.GetBlockToken(),
			},
			ClientName: proto.String(ClientName),
		},
		Offset: proto.Uint64(offset),
		Len:    proto.Uint64(length),
	}
}
<commit_msg>make sure we return io.ErrUnexpectedEOF if we get an EOF early<commit_after>package rpc

import (
	"bufio"
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"encoding/binary"
	"errors"
	"fmt"
	hdfs "github.com/colinmarc/hdfs/protocol/hadoop_hdfs"
	"hash/crc32"
	"io"
	"io/ioutil"
	"math"
	"net"
)

const (
	dataTransferVersion = 0x1c
	readBlockOp         = 0x51
)

// TODO: datanode blacklisting

// BlockReader implements io.ReaderCloser for reading a single block from HDFS,
// abstracting over reading from multiple datanodes.
type BlockReader struct {
	closed bool
	conn   net.Conn
	reader *bufio.Reader

	block       *hdfs.LocatedBlockProto
	checksumTab *crc32.Table

	offset    uint64
	chunkSize uint32
	packet    openPacket
	buf       bytes.Buffer
}

type openPacket struct {
	numChunks     int
	nextChunk     int
	checksumBytes []byte
	blockOffset   uint64
	packetOffset  uint64
	length        uint64
	last          bool
}

// NewBlockReader returns a new BlockReader, given the block information and
// security token from the namenode.
func NewBlockReader(block *hdfs.LocatedBlockProto, offset uint64) (*BlockReader, error) {
	br := &BlockReader{
		block:  block,
		offset: offset,
	}

	// TODO check multiple datanodes
	datanode := br.block.GetLocs()[0].GetId()
	address := fmt.Sprintf("%s:%d", datanode.GetIpAddr(), datanode.GetXferPort())
	err := br.connect(address)
	if err != nil {
		return nil, err
	}

	return br, nil
}

func (br *BlockReader) connect(datanode string) error {
	conn, err := net.DialTimeout("tcp", datanode, connectionTimeout)
	if err != nil {
		return err
	}

	br.conn = conn
	err = br.writeBlockReadRequest()
	if err != nil {
		return err
	}

	br.reader = bufio.NewReader(br.conn)
	resp, err := br.readBlockReadResponse()
	if err != nil {
		return err
	}

	checksumInfo := resp.GetReadOpChecksumInfo().GetChecksum()
	checksumType := checksumInfo.GetType()
	if checksumType == hdfs.ChecksumTypeProto_CHECKSUM_CRC32 {
		br.checksumTab = crc32.IEEETable
	} else if checksumType == hdfs.ChecksumTypeProto_CHECKSUM_CRC32C {
		br.checksumTab = crc32.MakeTable(crc32.Castagnoli)
	} else {
		return fmt.Errorf("Unsupported checksum type:", checksumType)
	}

	br.chunkSize = checksumInfo.GetBytesPerChecksum()
	br.startNewPacket()

	// The read will start aligned to a chunk boundary, so we need to seek forward
	// to the requested offset.
	amountToDiscard := br.offset - br.packet.blockOffset
	if amountToDiscard > 0 {
		io.CopyN(ioutil.Discard, br, int64(amountToDiscard))
	}

	return nil
}

func (br *BlockReader) Close() {
	br.conn.Close()
	br.closed = true
}

func (br *BlockReader) Read(b []byte) (int, error) {
	if br.closed {
		return 0, errors.New("The BlockReader is closed.")
	} else if br.offset >= br.block.GetB().GetNumBytes() {
		br.Close()
		return 0, io.EOF
	}

	// first, read any leftover data from buf
	if br.buf.Len() > 0 {
		n, _ := br.buf.Read(b)
		return n, nil
	}

	if br.packet.nextChunk >= br.packet.numChunks {
		br.startNewPacket()
	}

	// then, read until we fill up b or we reach the end of the packet
	readOffset := 0
	for br.packet.nextChunk < br.packet.numChunks {
		chOff := 4 * br.packet.nextChunk
		checksum := br.packet.checksumBytes[chOff : chOff+4]

		remaining := br.packet.length - br.packet.packetOffset
		chunkLength := int64(math.Min(float64(br.chunkSize), float64(remaining)))

		chunkReader := io.LimitReader(br.reader, int64(chunkLength))
		chunkBytes := b[readOffset:]
		n, err := chunkReader.Read(chunkBytes)

		readOffset += n
		br.packet.packetOffset += uint64(n)
		br.packet.nextChunk++

		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}

			br.Close()
			return readOffset, err
		}

		crc := crc32.Checksum(chunkBytes[:n], br.checksumTab)

		if int64(n) < chunkLength {
			// save any leftovers
			br.buf.Reset()
			leftover, err := br.buf.ReadFrom(chunkReader)
			if err != nil {
				return readOffset, err
			}

			br.packet.packetOffset += uint64(leftover)

			// update the checksum with the leftovers
			crc = crc32.Update(crc, br.checksumTab, br.buf.Bytes())
		}

		if crc != binary.BigEndian.Uint32(checksum) {
			return readOffset, errors.New("Invalid checksum from the datanode!")
		}

		if readOffset == len(b) {
			break
		}
	}

	return readOffset, nil
}

// A read request to a datanode:
// +-----------------------------------------------------------+
// |  Data Transfer Protocol Version, int16                    |
// +-----------------------------------------------------------+
// |  Op code, 1 byte (READ_BLOCK = 0x51)                      |
// +-----------------------------------------------------------+
// |  varint length + OpReadBlockProto                         |
// +-----------------------------------------------------------+
func (br *BlockReader) writeBlockReadRequest() error {
	header := []byte{0x00, dataTransferVersion, readBlockOp}

	// TODO offset/length?
	needed := (br.block.GetB().GetNumBytes() - br.offset)
	op := newReadBlockOp(br.block, br.offset, needed)
	opBytes, err := makeDelimitedMsg(op)
	if err != nil {
		return err
	}

	req := append(header, opBytes...)
	_, err = br.conn.Write(req)
	if err != nil {
		return err
	}

	return nil
}

// The initial response from the datanode:
// +-----------------------------------------------------------+
// |  varint length + BlockOpResponseProto                     |
// +-----------------------------------------------------------+
func (br *BlockReader) readBlockReadResponse() (*hdfs.BlockOpResponseProto, error) {
	respLength, err := binary.ReadUvarint(br.reader)
	if err != nil {
		if err != io.EOF {
			err = io.ErrUnexpectedEOF
		}

		return nil, err
	}

	respBytes := make([]byte, respLength)
	_, err = io.ReadFull(br.reader, respBytes)
	if err != nil {
		return nil, err
	}

	resp := &hdfs.BlockOpResponseProto{}
	err = proto.Unmarshal(respBytes, resp)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

// A packet from the datanode:
// +-----------------------------------------------------------+
// |  uint32 length of the packet                              |
// +-----------------------------------------------------------+
// |  size of the PacketHeaderProto, uint16                    |
// +-----------------------------------------------------------+
// |  PacketHeaderProto                                        |
// +-----------------------------------------------------------+
// |  N checksums, 4 bytes each                                |
// +-----------------------------------------------------------+
// |  N chunks of payload data                                 |
// +-----------------------------------------------------------+
func (br *BlockReader) startNewPacket() error {
	header, err := br.readPacketHeader()
	if err != nil {
		return err
	}

	blockOffset := uint64(header.GetOffsetInBlock())
	dataLength := uint64(header.GetDataLen())
	numChunks := int(math.Ceil(float64(dataLength) / float64(br.chunkSize)))

	// TODO don't assume checksum size is 4
	br.packet = openPacket{
		numChunks:     numChunks,
		nextChunk:     0,
		checksumBytes: make([]byte, numChunks*4),
		blockOffset:   blockOffset,
		packetOffset:  0,
		length:        dataLength,
		last:          header.GetLastPacketInBlock(),
	}

	_, err = io.ReadFull(br.reader, br.packet.checksumBytes)
	if err != nil {
		return err
	}

	return nil
}

func (br *BlockReader) readPacketHeader() (*hdfs.PacketHeaderProto, error) {
	var packetLength uint32
	err := binary.Read(br.reader, binary.BigEndian, &packetLength)
	if err != nil {
		if err != io.EOF {
			err = io.ErrUnexpectedEOF
		}

		return nil, err
	}

	var packetHeaderLength uint16
	err = binary.Read(br.reader, binary.BigEndian, &packetHeaderLength)
	if err != nil {
		if err != io.EOF {
			err = io.ErrUnexpectedEOF
		}

		return nil, err
	}

	packetHeaderBytes := make([]byte, packetHeaderLength)
	_, err = io.ReadFull(br.reader, packetHeaderBytes)
	if err != nil {
		return nil, err
	}

	packetHeader := &hdfs.PacketHeaderProto{}
	err = proto.Unmarshal(packetHeaderBytes, packetHeader)

	return packetHeader, nil
}

func newReadBlockOp(block *hdfs.LocatedBlockProto, offset, length uint64) *hdfs.OpReadBlockProto {
	return &hdfs.OpReadBlockProto{
		Header: &hdfs.ClientOperationHeaderProto{
			BaseHeader: &hdfs.BaseHeaderProto{
				Block: block.GetB(),
				Token: block.GetBlockToken(),
			},
			ClientName: proto.String(ClientName),
		},
		Offset: proto.Uint64(offset),
		Len:    proto.Uint64(length),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	if err != nil {
		//do nothing
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	// added by chenlin@20170308
	if len(args) != 0 {
		return nil, errors.New("incorrect args")
	}
	// adminCert, err := stub.GetCallerMetadata()
	// if err != nil {
	// 	return nil, errors.New("failed getting metadata")
	// }
	// if len(adminCert) == 0 {
	// 	return nil, errors.New("invalid admin certificate. Empty")
	// }
	stub.PutState("admin", []byte("hahahahaha"))
	err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
		&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
		&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_STRING, Key: false},
	})
	if err != nil {
		return nil, errors.New("Failed creating AssetsOnwership table.")
	}
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "putState":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		value := []byte(args[1])
		err := stub.PutState(key, value)
		if err != nil {
			return nil, err
		}
		return value, err
	case "delState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		err := stub.DelState(key)
		if err != nil {
			return nil, err
		}
		result := key + "has been deleted"
		return []byte(result), err

	case "createTable":
		err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
			&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
			&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_STRING, Key: false},
		})
		if err != nil {
			return nil, errors.New("Failed creating AssetsOnwership table.")
		}
		return nil, nil

	case "insertTable":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		owner := args[1]
		row := shim.Row{
			Columns: []*shim.Column{
				&shim.Column{Value: &shim.Column_String_{String_: asset}},
				&shim.Column{Value: &shim.Column_String_{String_: owner}}}}
		ok, err := stub.InsertRow("AssetsOwnership", row)
		if !ok && err == nil {
			return nil, errors.New("asset was already assigned")
		}
		return nil, nil //end of insertTable

	}
	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "getTxID":
		txID := stub.GetTxID()
		result := []byte(txID)
		return result, nil
	case "getTxTimestamp":
		time, err := stub.GetTxTimestamp()
		result := []byte(time.String()) //时间转换为字符串，time.String()
		return result, err
	case "getStringArgs":
		strList := stub.GetStringArgs()
		var result string
		for index := 0; index < len(strList); index++ {
			result += "***" + strList[index] + "***"
		}
		return []byte(result), nil

	case "getState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		result, err := stub.GetState(key)

		if err != nil {
			return nil, errors.New("Failed in function getState")
		}
		return result, err

	case "getCallerCert":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerCertificate()
		if err != nil {
			return nil, errors.New("Failed in function getCallerCert")
		}
		return result, err

	case "getCallerMetadata":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerMetadata()
		if err != nil {
			return nil, errors.New("Failed in function getCallerMetadata")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getCallerMetadata"), err
	case "getBinding":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetBinding()

		if err != nil {
			return nil, errors.New("Failed in function getBinding")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getBinding"), err
	case "getPayload":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetPayload()

		if err != nil {
			return nil, errors.New("Failed in function getPayload")
		}
		fmt.Println(result)
		return result, err
	// return []byte("getPayload"), err
	case "getRow":
		fmt.Println("*********************")
		fmt.Println("*********************")
		fmt.Println("*********************")
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		var columns []shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: asset}}
		columns = append(columns, col1)

		row, err := stub.GetRow("AssetsOwnership", columns)
		if err != nil {
			return nil, errors.New("failed in function getRow")
		}

		val0 := row.Columns[0].GetString_()
		val1 := row.Columns[1].GetString_()
		result := "****" + val0 + "****" + val1
		return []byte(result), err
	//end of "getRow"

	case "getRowDefinition":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		owner := args[1]

		col0 := shim.Column{Value: &shim.Column_String_{String_: asset}} //col0，type=shim.column
		col1 := shim.Column{Value: &shim.Column_String_{String_: owner}} //col1, type=shim.column
		var columns []shim.Column                                        //define columns=[]shim.column
		columns = append(columns, col0)
		columns = append(columns, col1) //完成columns的赋值

		val0 := columns[0].GetString_()
		val1 := columns[1].GetString_()
		result := val0 + "****" + val1
		return []byte(result), nil
	}

	return nil, nil
}
<commit_msg>demo05.go +++++ function getRows()<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	if err != nil {
		//do nothing
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	// added by chenlin@20170308
	if len(args) != 0 {
		return nil, errors.New("incorrect args")
	}
	// adminCert, err := stub.GetCallerMetadata()
	// if err != nil {
	// 	return nil, errors.New("failed getting metadata")
	// }
	// if len(adminCert) == 0 {
	// 	return nil, errors.New("invalid admin certificate. Empty")
	// }
	stub.PutState("admin", []byte("hahahahaha"))
	err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
		&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
		&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_STRING, Key: false},
	})
	if err != nil {
		return nil, errors.New("Failed creating AssetsOnwership table.")
	}
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "putState":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		value := []byte(args[1])
		err := stub.PutState(key, value)
		if err != nil {
			return nil, err
		}
		return value, err
	case "delState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		err := stub.DelState(key)
		if err != nil {
			return nil, err
		}
		result := key + "has been deleted"
		return []byte(result), err

	case "createTable":
		err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
			&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
			&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_STRING, Key: false},
		})
		if err != nil {
			return nil, errors.New("Failed creating AssetsOnwership table.")
		}
		return nil, nil

	case "insertTable":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		owner := args[1]
		row := shim.Row{
			Columns: []*shim.Column{
				&shim.Column{Value: &shim.Column_String_{String_: asset}},
				&shim.Column{Value: &shim.Column_String_{String_: owner}}}}
		ok, err := stub.InsertRow("AssetsOwnership", row)
		if !ok && err == nil {
			return nil, errors.New("asset was already assigned")
		}
		return nil, nil //end of insertTable

	}
	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "getTxID":
		txID := stub.GetTxID()
		result := []byte(txID)
		return result, nil
	case "getTxTimestamp":
		time, err := stub.GetTxTimestamp()
		result := []byte(time.String()) //时间转换为字符串，time.String()
		return result, err
	case "getStringArgs":
		strList := stub.GetStringArgs()
		var result string
		for index := 0; index < len(strList); index++ {
			result += "***" + strList[index] + "***"
		}
		return []byte(result), nil

	case "getState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		result, err := stub.GetState(key)

		if err != nil {
			return nil, errors.New("Failed in function getState")
		}
		return result, err

	case "getCallerCert":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerCertificate()
		if err != nil {
			return nil, errors.New("Failed in function getCallerCert")
		}
		return result, err

	case "getCallerMetadata":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerMetadata()
		if err != nil {
			return nil, errors.New("Failed in function getCallerMetadata")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getCallerMetadata"), err
	case "getBinding":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetBinding()

		if err != nil {
			return nil, errors.New("Failed in function getBinding")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getBinding"), err
	case "getPayload":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetPayload()

		if err != nil {
			return nil, errors.New("Failed in function getPayload")
		}
		fmt.Println(result)
		return result, err
	// return []byte("getPayload"), err
	case "getRow":
		fmt.Println("*********************")
		fmt.Println("*********************")
		fmt.Println("*********************")
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		var columns []shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: asset}}
		columns = append(columns, col1)

		row, err := stub.GetRow("AssetsOwnership", columns)
		if err != nil {
			return nil, errors.New("failed in function getRow")
		}

		val0 := row.Columns[0].GetString_()
		val1 := row.Columns[1].GetString_()
		result := "****" + val0 + "****" + val1
		return []byte(result), err
	//end of "getRow"

	case "getRowDefinition":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		owner := args[1]

		col0 := shim.Column{Value: &shim.Column_String_{String_: asset}} //col0，type=shim.column
		col1 := shim.Column{Value: &shim.Column_String_{String_: owner}} //col1, type=shim.column
		var columns []shim.Column                                        //define columns=[]shim.column
		columns = append(columns, col0)
		columns = append(columns, col1) //完成columns的赋值

		val0 := columns[0].GetString_()
		val1 := columns[1].GetString_()
		result := val0 + "****" + val1
		return []byte(result), nil

	case "getRows":
		var columns []shim.Column
		col0 := shim.Column{
			Value: &shim.Column_String_{},
		}
		columns = append(columns, col0)

		rowChannel, err := stub.GetRows("AssetsOwnership", columns)
		var rows []shim.Row
		for {
			select {
			case row, ok := <-rowChannel:
				if !ok {
					rowChannel = nil
				} else {
					rows = append(rows, row)
				}
			}
			if rowChannel == nil {
				break
			}
		}
		jsonRows, err := json.Marshal(rows)
		if err != nil {
			return nil, fmt.Errorf("getRowsTableTwo operation failed. Error marshaling JSON: %s", err)
		}
		return jsonRows, nil //end of function getRows()
	}

	return nil, nil
}
<|endoftext|>


<commit_before>package observable

import (
	"sync"

	"github.com/reactivex/rxgo/handlers"
	"github.com/reactivex/rxgo/observer"
)

// transforms emitted items into observables and flattens them into single observable.
// maxInParallel argument controls how many transformed observables are processed in parallel
// For an example please take a look at flatmap_slice_test.go file in the examples directory.
func (o Observable) FlatMap(apply func(interface{}) Observable, maxInParallel uint) Observable {
	return o.flatMap(apply, maxInParallel, flatObservedSequence)
}

func (o Observable) flatMap(
	apply func(interface{}) Observable,
	maxInParallel uint,
	flatteningFunc func(out chan interface{}, o Observable, apply func(interface{}) Observable, maxInParallel uint)) Observable {

	out := make(chan interface{})

	if maxInParallel < 1 {
		maxInParallel = 1
	}

	go flatteningFunc(out, o, apply, maxInParallel)

	return Observable(out)
}

func flatObservedSequence(out chan interface{}, o Observable, apply func(interface{}) Observable, maxInParallel uint) {
	var (
		sequence Observable
		wg       sync.WaitGroup
		count    uint
	)

	defer close(out)
	emissionObserver := newFlattenEmissionObserver(out)

	count = 0
	for element := range o {
		sequence = apply(element)
		count++
		wg.Add(1)
		go func() {
			defer wg.Done()
			<-(sequence.Subscribe(*emissionObserver))
		}()

		if count%maxInParallel == 0 {
			wg.Wait()
		}
	}

	wg.Wait()
}

func newFlattenEmissionObserver(out chan interface{}) *observer.Observer {
	ob := observer.New(handlers.NextFunc(func(element interface{}) {
		out <- element
	}))
	return &ob
}
<commit_msg>Fix flatmap race condition using incorrect observable from apply()<commit_after>package observable

import (
	"sync"

	"github.com/reactivex/rxgo/handlers"
	"github.com/reactivex/rxgo/observer"
)

// transforms emitted items into observables and flattens them into single observable.
// maxInParallel argument controls how many transformed observables are processed in parallel
// For an example please take a look at flatmap_slice_test.go file in the examples directory.
func (o Observable) FlatMap(apply func(interface{}) Observable, maxInParallel uint) Observable {
	return o.flatMap(apply, maxInParallel, flatObservedSequence)
}

func (o Observable) flatMap(
	apply func(interface{}) Observable,
	maxInParallel uint,
	flatteningFunc func(out chan interface{}, o Observable, apply func(interface{}) Observable, maxInParallel uint)) Observable {

	out := make(chan interface{})

	if maxInParallel < 1 {
		maxInParallel = 1
	}

	go flatteningFunc(out, o, apply, maxInParallel)

	return Observable(out)
}

func flatObservedSequence(out chan interface{}, o Observable, apply func(interface{}) Observable, maxInParallel uint) {
	var (
		sequence Observable
		wg       sync.WaitGroup
		count    uint
	)

	defer close(out)
	emissionObserver := newFlattenEmissionObserver(out)

	count = 0
	for element := range o {
		sequence = apply(element)
		count++
		wg.Add(1)
		go func(copy Observable) {
			defer wg.Done()
			<-(copy.Subscribe(*emissionObserver))
		}(sequence)

		if count%maxInParallel == 0 {
			wg.Wait()
		}
	}

	wg.Wait()
}

func newFlattenEmissionObserver(out chan interface{}) *observer.Observer {
	ob := observer.New(handlers.NextFunc(func(element interface{}) {
		out <- element
	}))
	return &ob
}
<|endoftext|>


<commit_before>// Copyright (C) 2013 Miquel Sabaté Solà
// This file is licensed under the MIT license.
// See the LICENSE file.

// This package encapsulates all the methods regarding the File Cache.
package fcache

import (
    "os"
    "fmt"
    "path"
    "time"
    "errors"
    "io/ioutil"
)

// This type contains some needed info that will be used when caching.
type Cache struct {
    // The directory
    Dir string

    // The expiration time to be set for each file.
    Expiration time.Duration

    // The permissions to be set when this cache creates new files.
    Permissions os.FileMode
}

// Get a pointer to an initialized Cache structure.
//
// dir        - The path to the cache directory. If the directory does not
//              exist, it will create a new directory with permissions 0644.
// expiration - The expiration time. That is, how many nanoseconds has to pass
//              by when a cache file is no longer considered valid.
// perm       - The permissions that the cache should operate in when creating
//              new files.
//
// Returns a Cache pointer that points to an initialized Cache structure. It
// will return nil if something goes wrong.
func NewCache(dir string, expiration time.Duration, perm os.FileMode) *Cache {
    // First of all, get the directory path straight.
    if _, err := os.Stat(dir); err != nil {
        if os.IsNotExist(err) {
            if err = os.MkdirAll(dir, perm); err != nil {
                fmt.Printf("Error: %v\n", err)
                return nil
            }
        } else {
            fmt.Printf("Error: %v\n", err)
            return nil
        }
    }

    // Now it's safe to create the cache.
    cache := new(Cache)
    cache.Dir = dir
    cache.Expiration = expiration
    cache.Permissions = perm
    return cache
}

// Set the contents for a cache file. If this file doesn't exist already, it
// will be created with permissions 0644.
//
// name     - The name of the file.
// contents - The contents that the cache file has to contain after calling
//            this function.
//
// Returns nil if everything was ok. Otherwise it will return an error.
func (c *Cache) Set(name string, contents []byte) error {
    url := path.Join(c.Dir, name)
    return ioutil.WriteFile(url, contents, c.Permissions)
}

// Get the contents of a valid cache file.
//
// name - The name of the file.
//
// Returns a slice of bytes and an error. The slice of bytes contain the
// contents of the cache file. The error is set to nil if everything was fine.
func (c *Cache) Get(name string) ([]byte, error) {
    url := path.Join(c.Dir, name)
    if fi, err := os.Stat(url); err == nil {
        elapsed := time.Now().Sub(fi.ModTime())
        if c.Expiration > elapsed {
            return ioutil.ReadFile(url)
        }
        // Remove this file, its time has expired.
        os.Remove(url)
    }
    return []byte{}, errors.New("miss.")
}

// Remove a cache file.
//
// name - The name of the file.
//
// Returns nil if everything was ok. Otherwise it will return an error.
func (c *Cache) Flush(name string) error {
    url := path.Join(c.Dir, name)
    return os.Remove(url)
}

// Remove all the files from the cache.
//
// Returns nil if everything was ok. Otherwise it will return an error.
func (c *Cache) FlushAll() error {
    url := path.Join(c.Dir)
    err := os.RemoveAll(url)
    if err == nil {
        err = os.MkdirAll(url, c.Permissions)
    }
    return err
}
<commit_msg>Removing old documentation.<commit_after>// Copyright (C) 2013 Miquel Sabaté Solà
// This file is licensed under the MIT license.
// See the LICENSE file.

// This package encapsulates all the methods regarding the File Cache.
package fcache

import (
    "os"
    "fmt"
    "path"
    "time"
    "errors"
    "io/ioutil"
)

// This type contains some needed info that will be used when caching.
type Cache struct {
    // The directory.
    Dir string

    // The expiration time to be set for each file.
    Expiration time.Duration

    // The permissions to be set when this cache creates new files.
    Permissions os.FileMode
}

// Get a pointer to an initialized Cache structure.
//
// dir        - The path to the cache directory. If the directory does not
//              exist, it will create it for you.
// expiration - The expiration time. That is, how many nanoseconds has to pass
//              by when a cache file is no longer considered valid.
// perm       - The permissions that the cache should operate in when creating
//              new files.
//
// Returns a Cache pointer that points to an initialized Cache structure. It
// will return nil if something goes wrong.
func NewCache(dir string, expiration time.Duration, perm os.FileMode) *Cache {
    // First of all, get the directory path straight.
    if _, err := os.Stat(dir); err != nil {
        if os.IsNotExist(err) {
            if err = os.MkdirAll(dir, perm); err != nil {
                fmt.Printf("Error: %v\n", err)
                return nil
            }
        } else {
            fmt.Printf("Error: %v\n", err)
            return nil
        }
    }

    // Now it's safe to create the cache.
    cache := new(Cache)
    cache.Dir = dir
    cache.Expiration = expiration
    cache.Permissions = perm
    return cache
}

// Set the contents for a cache file. If this file doesn't exist already, it
// will be created for you.
//
// name     - The name of the file.
// contents - The contents that the cache file has to contain after calling
//            this function.
//
// Returns nil if everything was ok. Otherwise it will return an error.
func (c *Cache) Set(name string, contents []byte) error {
    url := path.Join(c.Dir, name)
    return ioutil.WriteFile(url, contents, c.Permissions)
}

// Get the contents of a valid cache file.
//
// name - The name of the file.
//
// Returns a slice of bytes and an error. The slice of bytes contain the
// contents of the cache file. The error is set to nil if everything was fine.
func (c *Cache) Get(name string) ([]byte, error) {
    url := path.Join(c.Dir, name)
    if fi, err := os.Stat(url); err == nil {
        elapsed := time.Now().Sub(fi.ModTime())
        if c.Expiration > elapsed {
            return ioutil.ReadFile(url)
        }
        // Remove this file, its time has expired.
        os.Remove(url)
    }
    return []byte{}, errors.New("miss.")
}

// Remove a cache file.
//
// name - The name of the file.
//
// Returns nil if everything was ok. Otherwise it will return an error.
func (c *Cache) Flush(name string) error {
    url := path.Join(c.Dir, name)
    return os.Remove(url)
}

// Remove all the files from the cache.
//
// Returns nil if everything was ok. Otherwise it will return an error.
func (c *Cache) FlushAll() error {
    url := path.Join(c.Dir)
    err := os.RemoveAll(url)
    if err == nil {
        err = os.MkdirAll(url, c.Permissions)
    }
    return err
}
<|endoftext|>


<commit_before>package layout

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"

	"github.com/containers/image/v5/manifest"
	"github.com/containers/image/v5/pkg/tlsclientconfig"
	"github.com/containers/image/v5/types"
	"github.com/docker/go-connections/tlsconfig"
	"github.com/opencontainers/go-digest"
	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type ociImageSource struct {
	ref           ociReference
	index         *imgspecv1.Index
	descriptor    imgspecv1.Descriptor
	client        *http.Client
	sharedBlobDir string
}

// newImageSource returns an ImageSource for reading from an existing directory.
func newImageSource(sys *types.SystemContext, ref ociReference) (types.ImageSource, error) {
	tr := tlsclientconfig.NewTransport()
	tr.TLSClientConfig = tlsconfig.ServerDefault()

	if sys != nil && sys.OCICertPath != "" {
		if err := tlsclientconfig.SetupCertificates(sys.OCICertPath, tr.TLSClientConfig); err != nil {
			return nil, err
		}
		tr.TLSClientConfig.InsecureSkipVerify = sys.OCIInsecureSkipTLSVerify
	}

	client := &http.Client{}
	client.Transport = tr
	descriptor, err := ref.getManifestDescriptor()
	if err != nil {
		return nil, err
	}
	index, err := ref.getIndex()
	if err != nil {
		return nil, err
	}
	d := &ociImageSource{ref: ref, index: index, descriptor: descriptor, client: client}
	if sys != nil {
		// TODO(jonboulle): check dir existence?
		d.sharedBlobDir = sys.OCISharedBlobDirPath
	}
	return d, nil
}

// Reference returns the reference used to set up this source.
func (s *ociImageSource) Reference() types.ImageReference {
	return s.ref
}

// Close removes resources associated with an initialized ImageSource, if any.
func (s *ociImageSource) Close() error {
	return nil
}

// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
	var dig digest.Digest
	var mimeType string
	var err error

	if instanceDigest == nil {
		dig = digest.Digest(s.descriptor.Digest)
		mimeType = s.descriptor.MediaType
	} else {
		dig = *instanceDigest
		for _, md := range s.index.Manifests {
			if md.Digest == dig {
				mimeType = md.MediaType
				break
			}
		}
	}

	manifestPath, err := s.ref.blobPath(dig, s.sharedBlobDir)
	if err != nil {
		return nil, "", err
	}

	m, err := ioutil.ReadFile(manifestPath)
	if err != nil {
		logrus.Errorf("Error HERE")
		return nil, "", err
	}
	if mimeType == "" {
		mimeType = manifest.GuessMIMEType(m)
	}

	return m, mimeType, nil
}

// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently.
func (s *ociImageSource) HasThreadSafeGetBlob() bool {
	return false
}

// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *ociImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
	if len(info.URLs) != 0 {
		return s.getExternalBlob(ctx, info.URLs)
	}

	path, err := s.ref.blobPath(info.Digest, s.sharedBlobDir)
	if err != nil {
		return nil, 0, err
	}

	r, err := os.Open(path)
	if err != nil {
		return nil, 0, err
	}
	fi, err := r.Stat()
	if err != nil {
		return nil, 0, err
	}
	return r, fi.Size(), nil
}

// GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for
// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
// (e.g. if the source never returns manifest lists).
func (s *ociImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) {
	return [][]byte{}, nil
}

func (s *ociImageSource) getExternalBlob(ctx context.Context, urls []string) (io.ReadCloser, int64, error) {
	if len(urls) == 0 {
		return nil, 0, errors.New("internal error: getExternalBlob called with no URLs")
	}

	errWrap := errors.New("failed fetching external blob from all urls")
	for _, url := range urls {

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error())
			continue
		}

		resp, err := s.client.Do(req.WithContext(ctx))
		if err != nil {
			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error())
			continue
		}

		if resp.StatusCode != http.StatusOK {
			resp.Body.Close()
			errWrap = errors.Wrapf(errWrap, "fetching %s failed, response code not 200", url)
			continue
		}

		return resp.Body, getBlobSize(resp), nil
	}

	return nil, 0, errWrap
}

// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer
// blobsums that are listed in the image's manifest.  If values are returned, they should be used when using GetBlob()
// to read the image's layers.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for
// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
// (e.g. if the source never returns manifest lists).
// The Digest field is guaranteed to be provided; Size may be -1.
// WARNING: The list may contain duplicates, and they are semantically relevant.
func (s *ociImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) {
	return nil, nil
}

func getBlobSize(resp *http.Response) int64 {
	size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
	if err != nil {
		size = -1
	}
	return size
}
<commit_msg>oci/layout: remove debug log<commit_after>package layout

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"

	"github.com/containers/image/v5/manifest"
	"github.com/containers/image/v5/pkg/tlsclientconfig"
	"github.com/containers/image/v5/types"
	"github.com/docker/go-connections/tlsconfig"
	"github.com/opencontainers/go-digest"
	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type ociImageSource struct {
	ref           ociReference
	index         *imgspecv1.Index
	descriptor    imgspecv1.Descriptor
	client        *http.Client
	sharedBlobDir string
}

// newImageSource returns an ImageSource for reading from an existing directory.
func newImageSource(sys *types.SystemContext, ref ociReference) (types.ImageSource, error) {
	tr := tlsclientconfig.NewTransport()
	tr.TLSClientConfig = tlsconfig.ServerDefault()

	if sys != nil && sys.OCICertPath != "" {
		if err := tlsclientconfig.SetupCertificates(sys.OCICertPath, tr.TLSClientConfig); err != nil {
			return nil, err
		}
		tr.TLSClientConfig.InsecureSkipVerify = sys.OCIInsecureSkipTLSVerify
	}

	client := &http.Client{}
	client.Transport = tr
	descriptor, err := ref.getManifestDescriptor()
	if err != nil {
		return nil, err
	}
	index, err := ref.getIndex()
	if err != nil {
		return nil, err
	}
	d := &ociImageSource{ref: ref, index: index, descriptor: descriptor, client: client}
	if sys != nil {
		// TODO(jonboulle): check dir existence?
		d.sharedBlobDir = sys.OCISharedBlobDirPath
	}
	return d, nil
}

// Reference returns the reference used to set up this source.
func (s *ociImageSource) Reference() types.ImageReference {
	return s.ref
}

// Close removes resources associated with an initialized ImageSource, if any.
func (s *ociImageSource) Close() error {
	return nil
}

// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
// It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
	var dig digest.Digest
	var mimeType string
	var err error

	if instanceDigest == nil {
		dig = digest.Digest(s.descriptor.Digest)
		mimeType = s.descriptor.MediaType
	} else {
		dig = *instanceDigest
		for _, md := range s.index.Manifests {
			if md.Digest == dig {
				mimeType = md.MediaType
				break
			}
		}
	}

	manifestPath, err := s.ref.blobPath(dig, s.sharedBlobDir)
	if err != nil {
		return nil, "", err
	}

	m, err := ioutil.ReadFile(manifestPath)
	if err != nil {
		return nil, "", err
	}
	if mimeType == "" {
		mimeType = manifest.GuessMIMEType(m)
	}

	return m, mimeType, nil
}

// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently.
func (s *ociImageSource) HasThreadSafeGetBlob() bool {
	return false
}

// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *ociImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
	if len(info.URLs) != 0 {
		return s.getExternalBlob(ctx, info.URLs)
	}

	path, err := s.ref.blobPath(info.Digest, s.sharedBlobDir)
	if err != nil {
		return nil, 0, err
	}

	r, err := os.Open(path)
	if err != nil {
		return nil, 0, err
	}
	fi, err := r.Stat()
	if err != nil {
		return nil, 0, err
	}
	return r, fi.Size(), nil
}

// GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for
// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
// (e.g. if the source never returns manifest lists).
func (s *ociImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) {
	return [][]byte{}, nil
}

func (s *ociImageSource) getExternalBlob(ctx context.Context, urls []string) (io.ReadCloser, int64, error) {
	if len(urls) == 0 {
		return nil, 0, errors.New("internal error: getExternalBlob called with no URLs")
	}

	errWrap := errors.New("failed fetching external blob from all urls")
	for _, url := range urls {

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error())
			continue
		}

		resp, err := s.client.Do(req.WithContext(ctx))
		if err != nil {
			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error())
			continue
		}

		if resp.StatusCode != http.StatusOK {
			resp.Body.Close()
			errWrap = errors.Wrapf(errWrap, "fetching %s failed, response code not 200", url)
			continue
		}

		return resp.Body, getBlobSize(resp), nil
	}

	return nil, 0, errWrap
}

// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer
// blobsums that are listed in the image's manifest.  If values are returned, they should be used when using GetBlob()
// to read the image's layers.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for
// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list
// (e.g. if the source never returns manifest lists).
// The Digest field is guaranteed to be provided; Size may be -1.
// WARNING: The list may contain duplicates, and they are semantically relevant.
func (s *ociImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) {
	return nil, nil
}

func getBlobSize(resp *http.Response) int64 {
	size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
	if err != nil {
		size = -1
	}
	return size
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/miekg/dns"
    "strings"
)

type QueryFilter struct {
    domain          string
    qTypes          []string
}

type QueryFilterer struct {
    acceptFilters   []QueryFilter
    rejectFilters   []QueryFilter
}

// Matches returns true if the given DNS query matches the filter
func (f *QueryFilter) Matches(req *dns.Msg) bool {
    queryDomain := req.Question[0].Name
    queryQType := dns.TypeToString[req.Question[0].Qtype]
    if len(queryDomain) > 0 && !strings.HasSuffix(queryDomain, f.domain) {
        debugMsg("Domain match failed (" + queryDomain + ", " + f.domain + ")")
        return false
    }

    matches := false
    if len(f.qTypes) > 0 {
        for _, qType := range f.qTypes {
            if qType == queryQType {
                matches = true
            }
        }
    } else {
        matches = true
    }

    return matches
}

// ShouldAcceptQuery returns true if the given DNS query matches the given
// accept/reject filters, and should be accepted.
func (f *QueryFilterer) ShouldAcceptQuery(req *dns.Msg) bool {
    accepted := true

    for _, filter := range f.rejectFilters {
        filterDescription := "Filter " + filter.domain + ":" + strings.Join(filter.qTypes, ",")
        if filter.Matches(req) {
            debugMsg(filterDescription + " rejected")
            accepted = false
            break
        }
        debugMsg(filterDescription + " not rejected")
    }

    if accepted {
        for _, filter := range f.acceptFilters {
            filterDescription := "Filter " + filter.domain + ":" + strings.Join(filter.qTypes, ",")
            if !filter.Matches(req){
                debugMsg(filterDescription + " not accepted")
                accepted = false
                break
            }
            debugMsg(filterDescription + " accepted")
        }
    }

    return accepted
}
<commit_msg>Changed the query filter check to match when no Accept filters are provided<commit_after>package main

import (
    "github.com/miekg/dns"
    "strings"
)

type QueryFilter struct {
    domain          string
    qTypes          []string
}

type QueryFilterer struct {
    acceptFilters   []QueryFilter
    rejectFilters   []QueryFilter
}

// Matches returns true if the given DNS query matches the filter
func (f *QueryFilter) Matches(req *dns.Msg) bool {
    queryDomain := req.Question[0].Name
    queryQType := dns.TypeToString[req.Question[0].Qtype]
    if len(queryDomain) > 0 && !strings.HasSuffix(queryDomain, f.domain) {
        debugMsg("Domain match failed (" + queryDomain + ", " + f.domain + ")")
        return false
    }

    matches := false
    if len(f.qTypes) > 0 {
        for _, qType := range f.qTypes {
            if qType == queryQType {
                matches = true
            }
        }
    } else {
        matches = true
    }

    return matches
}

// ShouldAcceptQuery returns true if the given DNS query matches the given
// accept/reject filters, and should be accepted.
func (f *QueryFilterer) ShouldAcceptQuery(req *dns.Msg) bool {
    accepted := true

    for _, filter := range f.rejectFilters {
        filterDescription := "Filter " + filter.domain + ":" + strings.Join(filter.qTypes, ",")
        if filter.Matches(req) {
            debugMsg(filterDescription + " rejected")
            accepted = false
            break
        }
        debugMsg(filterDescription + " not rejected")
    }

    if accepted && len(f.acceptFilters) > 0 {
        accepted = false
        for _, filter := range f.acceptFilters {
            filterDescription := "Filter " + filter.domain + ":" + strings.Join(filter.qTypes, ",")
            if filter.Matches(req) {
                debugMsg(filterDescription + " accepted")
                accepted = true
                break
            }
            debugMsg(filterDescription + " not accepted")
        }
    }

    return accepted
}
<|endoftext|>


<commit_before>//Package gpsdfilter is for filtering gpsd JSON documents
package gpsdfilter

import (
	"encoding/json"
	"sync"
)

//Rule is a type that descripe how to filter a gpsd JSON document
type Rule struct {
	Class string
	DoLog bool
	Type  Type
}

//Filter contains all the filter rules of type *Rule
type Filter struct {
	mutex sync.Mutex
	rules map[string]*Rule
}

//AddRule adds a *Rule
//Error ErrRuleIsNil is returned if the *Rule is nil, otherwise
//the nil error is returned
func (f *Filter) AddRule(r *Rule) error {
	f.mutex.Lock()
	defer f.mutex.Unlock()

	if r == nil {
		return ErrRuleIsNil
	}
	f.rules[r.Class] = r

	return nil
}

//Filter takes a byte slice as input, and returns 3 types of information:
//  A boolean: If true, the gpsd JSON document should be logged, otherwise not logged
//  A Type: Tells what to do with the gpsd JSON document.
//  An error: Error ErrFilterNoSuchRule is returned if the rule is unknown.
//  An error from the json paser can also be returned, and
//  otherwise the nil error is returned.
func (f *Filter) Filter(p []byte) (bool, Type, error) {
	var (
		c   Class
		err error
	)
	if err = json.Unmarshal(p, &c); err != nil {
		return false, TypeUnknown, err
	}
	return f.FilterClass(c.Class)
}

//FilterClass takes a class of type string, fx. "TPV", as input, and returns 3
// types of information:
//  A boolean: If true, the gpsd JSON document should be logged, otherwise not logged
//  A Type: Tells what to do with the gpsd JSON document.
//  An error: Error ErrFilterNoSuchRule is returned if the rule is unknown.
//  An error from the json paser can also be ret
func (f *Filter) FilterClass(class string) (bool, Type, error) {
	var (
		rule *Rule
		ok   bool
	)
	f.mutex.Lock()
	defer f.mutex.Unlock()

	if rule, ok = f.rules[class]; ok == false {
		return false, TypeUnknown, ErrFilterNoSuchRule
	}
	return rule.DoLog, rule.Type, nil
}
<commit_msg> Fixed a documentation typo: 	modified:   filter.go<commit_after>//Package gpsdfilter is for filtering gpsd JSON documents
package gpsdfilter

import (
	"encoding/json"
	"sync"
)

//Rule is a type that descripe how to filter a gpsd JSON document
type Rule struct {
	Class string
	DoLog bool
	Type  Type
}

//Filter contains all the filter rules of type *Rule
type Filter struct {
	mutex sync.Mutex
	rules map[string]*Rule
}

//AddRule adds a *Rule
//Error ErrRuleIsNil is returned if the *Rule is nil, otherwise
//the nil error is returned
func (f *Filter) AddRule(r *Rule) error {
	f.mutex.Lock()
	defer f.mutex.Unlock()

	if r == nil {
		return ErrRuleIsNil
	}
	f.rules[r.Class] = r

	return nil
}

//Filter takes a byte slice as input, and returns 3 types of information:
//  A boolean: If true, the gpsd JSON document should be logged, otherwise not logged
//  A Type: Tells what to do with the gpsd JSON document.
//  An error: Error ErrFilterNoSuchRule is returned if the rule is unknown.
//  An error from the json paser can also be returned, and
//  otherwise the nil error is returned.
func (f *Filter) Filter(p []byte) (bool, Type, error) {
	var (
		c   Class
		err error
	)
	if err = json.Unmarshal(p, &c); err != nil {
		return false, TypeUnknown, err
	}
	return f.FilterClass(c.Class)
}

//FilterClass takes a class of type string, fx. "TPV", as input, and returns 3
// types of information:
//  A boolean: If true, the gpsd JSON document should be logged, otherwise not logged
//  A Type: Tells what to do with the gpsd JSON document.
//  An error: Error ErrFilterNoSuchRule is returned if the rule is unknown.
//  An error from the json paser can also be returned
func (f *Filter) FilterClass(class string) (bool, Type, error) {
	var (
		rule *Rule
		ok   bool
	)
	f.mutex.Lock()
	defer f.mutex.Unlock()

	if rule, ok = f.rules[class]; ok == false {
		return false, TypeUnknown, ErrFilterNoSuchRule
	}
	return rule.DoLog, rule.Type, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package controller provides a Kubernetes controller for a TFJob resource.
package controller

import (
	"fmt"
	"strconv"
	"strings"

	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"

	tfv1alpha2 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1alpha2"
)

// reconcilePods checks and updates pods for each given TFReplicaSpec.
// It will requeue the tfjob in case of an error while creating/deleting pods.
func (tc *TFJobController) reconcilePods(
	tfjob *tfv1alpha2.TFJob,
	pods []*v1.Pod,
	rtype tfv1alpha2.TFReplicaType,
	spec *tfv1alpha2.TFReplicaSpec) error {

	// Convert TFReplicaType to lower string.
	rt := strings.ToLower(string(rtype))
	// Get all pods for the type rt.
	pods = filterPodsForTFReplicaType(pods, rt)
	replicas := int(*spec.Replicas)

	initializeTFReplicaStatuses(tfjob, rtype)

	podSlices := getPodSlices(pods, replicas, loggerForReplica(tfjob, rt))
	for index, podSlice := range podSlices {
		if len(podSlice) > 1 {
			loggerForReplica(tfjob, rt).Warning("We have to many pods for the worker %d", index)
			// TODO(gaocegege): Kill some pods.
		} else if len(podSlice) == 0 {
			loggerForReplica(tfjob, rt).Infof("need to create new pod: %s-%d", rt, index)
			err := tc.createNewPod(tfjob, rt, string(index), spec)
			if err != nil {
				return err
			}
			increaseTFJobReplicaStatusesActive(tfjob, rtype)
		} else {
			// We already have one, and check the status.
			pod := podSlice[0]
			updateTFJobReplicaStatuses(tfjob, rtype, pod)
		}
	}

	return tc.updateStatus(tfjob, rtype, replicas)
}

// getPodSlices returns a slice, which element is the slice of pod.
func getPodSlices(pods []*v1.Pod, replicas int, logger *log.Entry) [][]*v1.Pod {
	podSlices := make([][]*v1.Pod, replicas)
	for _, pod := range pods {
		if _, ok := pod.Labels[tfReplicaIndexLabel]; !ok {
			logger.Warning("The pod do not have the index label.")
			continue
		}
		index, err := strconv.Atoi(pod.Labels[tfReplicaIndexLabel])
		if err != nil {
			logger.Warning("Error when strconv.Atoi: %v", err)
			continue
		}
		if index < 0 || index >= replicas {
			logger.Warningf("The label index is not expected: %d", index)
		} else {
			podSlices[index] = append(podSlices[index], pod)
		}
	}
	return podSlices
}

// createNewPod creates a new pod for the given index and type.
func (tc *TFJobController) createNewPod(tfjob *tfv1alpha2.TFJob, rt, index string, spec *tfv1alpha2.TFReplicaSpec) error {
	tfjobKey, err := KeyFunc(tfjob)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("Couldn't get key for tfjob object %#v: %v", tfjob, err))
		return err
	}
	expectationPodsKey := genExpectationPodsKey(tfjobKey, rt)
	err = tc.expectations.ExpectCreations(expectationPodsKey, 1)
	if err != nil {
		return err
	}

	// Create OwnerReference.
	controllerRef := genOwnerReference(tfjob)

	// Set type and index for the worker.
	labels := genLabels(tfjobKey)
	labels[tfReplicaTypeLabel] = rt
	labels[tfReplicaIndexLabel] = index

	podTemplate := spec.Template.DeepCopy()

	if podTemplate.Labels == nil {
		podTemplate.Labels = make(map[string]string)
	}

	for key, value := range labels {
		podTemplate.Labels[key] = value
	}

	// Generate TF_CONFIG JSON string.
	tfConfigStr := genTFConfigJSONStr(tfjob, rt, index)

	if tfConfigStr == "" {
		return nil
	}
	// Add TF_CONFIG environment variable.
	for i := range podTemplate.Spec.Containers {
		if len(podTemplate.Spec.Containers[i].Env) == 0 {
			podTemplate.Spec.Containers[i].Env = make([]v1.EnvVar, 0)
		}
		podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, v1.EnvVar{
			Name:  "TF_CONFIG",
			Value: tfConfigStr,
		})
	}

	// TODO(gaocegege): Deal with RestartPolicyExitCode.
	// Set restart policy
	if spec.RestartPolicy != tfv1alpha2.RestartPolicyExitCode {
		podTemplate.Spec.RestartPolicy = v1.RestartPolicy(spec.RestartPolicy)
	}

	err = tc.podControl.CreatePodsWithControllerRef(tfjob.Namespace, podTemplate, tfjob, controllerRef)
	if err != nil && errors.IsTimeout(err) {
		// Pod is created but its initialization has timed out.
		// If the initialization is successful eventually, the
		// controller will observe the creation via the informer.
		// If the initialization fails, or if the pod keeps
		// uninitialized for a long time, the informer will not
		// receive any update, and the controller will create a new
		// pod when the expectation expires.
		return nil
	} else if err != nil {
		return err
	}
	return nil
}

// getPodsForTFJob returns the set of pods that this tfjob should manage.
// It also reconciles ControllerRef by adopting/orphaning.
// Note that the returned Pods are pointers into the cache.
func (tc *TFJobController) getPodsForTFJob(tfjob *tfv1alpha2.TFJob) ([]*v1.Pod, error) {
	tfjobKey, err := KeyFunc(tfjob)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("Couldn't get key for tfjob object %#v: %v", tfjob, err))
		return nil, err
	}

	// Create selector.
	selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
		MatchLabels: genLabels(tfjobKey),
	})

	if err != nil {
		return nil, fmt.Errorf("couldn't convert Job selector: %v", err)
	}
	// List all pods to include those that don't match the selector anymore
	// but have a ControllerRef pointing to this controller.
	pods, err := tc.podLister.Pods(tfjob.Namespace).List(labels.Everything())
	if err != nil {
		return nil, err
	}

	// If any adoptions are attempted, we should first recheck for deletion
	// with an uncached quorum read sometime after listing Pods (see #42639).
	canAdoptFunc := RecheckDeletionTimestamp(func() (metav1.Object, error) {
		fresh, err := tc.tfJobClientSet.KubeflowV1alpha2().TFJobs(tfjob.Namespace).Get(tfjob.Name, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}
		if fresh.UID != tfjob.UID {
			return nil, fmt.Errorf("original TFJob %v/%v is gone: got uid %v, wanted %v", tfjob.Namespace, tfjob.Name, fresh.UID, tfjob.UID)
		}
		return fresh, nil
	})
	cm := NewPodControllerRefManager(tc.podControl, tfjob, selector, controllerKind, canAdoptFunc)
	return cm.ClaimPods(pods)
}

// filterPodsForTFReplicaType returns pods belong to a TFReplicaType.
func filterPodsForTFReplicaType(pods []*v1.Pod, tfReplicaType string) []*v1.Pod {
	var result []*v1.Pod

	tfReplicaSelector := &metav1.LabelSelector{
		MatchLabels: make(map[string]string),
	}

	tfReplicaSelector.MatchLabels[tfReplicaTypeLabel] = tfReplicaType

	for _, pod := range pods {
		selector, _ := metav1.LabelSelectorAsSelector(tfReplicaSelector)
		if !selector.Matches(labels.Set(pod.Labels)) {
			continue
		}
		result = append(result, pod)
	}
	return result
}

func genExpectationPodsKey(tfjobKey, replicaType string) string {
	return tfjobKey + "/" + strings.ToLower(replicaType) + "/pods"
}

// When a pod is created, enqueue the tfjob that manages it and update its expectations.
func (tc *TFJobController) addPod(obj interface{}) {
	pod := obj.(*v1.Pod)
	if pod.DeletionTimestamp != nil {
		// on a restart of the controller controller, it's possible a new pod shows up in a state that
		// is already pending deletion. Prevent the pod from being a creation observation.
		// tc.deletePod(pod)
		return
	}

	// If it has a ControllerRef, that's all that matters.
	if controllerRef := metav1.GetControllerOf(pod); controllerRef != nil {
		tfjob := tc.resolveControllerRef(pod.Namespace, controllerRef)
		if tfjob == nil {
			log.Info("This pod's tfjob does not exists")
			return
		}

		tfjobKey, err := KeyFunc(tfjob)
		if err != nil {
			loggerForTFJob(tfjob).Infof("Failed to get the key of the tfjob: %v", err)
			return
		}

		if _, ok := pod.Labels[tfReplicaTypeLabel]; !ok {
			loggerForTFJob(tfjob).Info("This pod maybe not created by tf-operator")
			return
		}

		rtype := pod.Labels[tfReplicaTypeLabel]
		expectationPodsKey := genExpectationPodsKey(tfjobKey, rtype)

		tc.expectations.CreationObserved(expectationPodsKey)
		tc.enqueueTFJob(tfjob)

		return
	}

	// Otherwise, it's an orphan. Get a list of all matching controllers and sync
	// them to see if anyone wants to adopt it.
	// DO NOT observe creation because no controller should be waiting for an
	// orphan.
	// for _, tfjob := range tc.getPodJobs(pod) {
	// 	tc.enqueueTFJob(tfjob)
	// }
}

// When a pod is updated, figure out what tfjob/s manage it and wake them up.
// If the labels of the pod have changed we need to awaken both the old
// and new replica set. old and cur must be *v1.Pod types.
func (tc *TFJobController) updatePod(old, cur interface{}) {
	// TODO(CPH): handle this gracefully.
}

// When a pod is deleted, enqueue the tfjob that manages the pod and update its expectations.
// obj could be an *v1.Pod, or a DeletionFinalStateUnknown marker item.
func (tc *TFJobController) deletePod(obj interface{}) {
	// TODO(CPH): handle this gracefully.
}
<commit_msg>[v1alpha2]fix bug int to string for index (#571)<commit_after>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package controller provides a Kubernetes controller for a TFJob resource.
package controller

import (
	"fmt"
	"strconv"
	"strings"

	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"

	tfv1alpha2 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1alpha2"
)

// reconcilePods checks and updates pods for each given TFReplicaSpec.
// It will requeue the tfjob in case of an error while creating/deleting pods.
func (tc *TFJobController) reconcilePods(
	tfjob *tfv1alpha2.TFJob,
	pods []*v1.Pod,
	rtype tfv1alpha2.TFReplicaType,
	spec *tfv1alpha2.TFReplicaSpec) error {

	// Convert TFReplicaType to lower string.
	rt := strings.ToLower(string(rtype))
	// Get all pods for the type rt.
	pods = filterPodsForTFReplicaType(pods, rt)
	replicas := int(*spec.Replicas)

	initializeTFReplicaStatuses(tfjob, rtype)

	podSlices := getPodSlices(pods, replicas, loggerForReplica(tfjob, rt))
	for index, podSlice := range podSlices {
		if len(podSlice) > 1 {
			loggerForReplica(tfjob, rt).Warning("We have to many pods for the worker %d", index)
			// TODO(gaocegege): Kill some pods.
		} else if len(podSlice) == 0 {
			loggerForReplica(tfjob, rt).Infof("need to create new pod: %s-%d", rt, index)
			err := tc.createNewPod(tfjob, rt, strconv.Itoa(index), spec)
			if err != nil {
				return err
			}
			increaseTFJobReplicaStatusesActive(tfjob, rtype)
		} else {
			// We already have one, and check the status.
			pod := podSlice[0]
			updateTFJobReplicaStatuses(tfjob, rtype, pod)
		}
	}

	return tc.updateStatus(tfjob, rtype, replicas)
}

// getPodSlices returns a slice, which element is the slice of pod.
func getPodSlices(pods []*v1.Pod, replicas int, logger *log.Entry) [][]*v1.Pod {
	podSlices := make([][]*v1.Pod, replicas)
	for _, pod := range pods {
		if _, ok := pod.Labels[tfReplicaIndexLabel]; !ok {
			logger.Warning("The pod do not have the index label.")
			continue
		}
		index, err := strconv.Atoi(pod.Labels[tfReplicaIndexLabel])
		if err != nil {
			logger.Warning("Error when strconv.Atoi: %v", err)
			continue
		}
		if index < 0 || index >= replicas {
			logger.Warningf("The label index is not expected: %d", index)
		} else {
			podSlices[index] = append(podSlices[index], pod)
		}
	}
	return podSlices
}

// createNewPod creates a new pod for the given index and type.
func (tc *TFJobController) createNewPod(tfjob *tfv1alpha2.TFJob, rt, index string, spec *tfv1alpha2.TFReplicaSpec) error {
	tfjobKey, err := KeyFunc(tfjob)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("Couldn't get key for tfjob object %#v: %v", tfjob, err))
		return err
	}
	expectationPodsKey := genExpectationPodsKey(tfjobKey, rt)
	err = tc.expectations.ExpectCreations(expectationPodsKey, 1)
	if err != nil {
		return err
	}

	// Create OwnerReference.
	controllerRef := genOwnerReference(tfjob)

	// Set type and index for the worker.
	labels := genLabels(tfjobKey)
	labels[tfReplicaTypeLabel] = rt
	labels[tfReplicaIndexLabel] = index

	podTemplate := spec.Template.DeepCopy()

	if podTemplate.Labels == nil {
		podTemplate.Labels = make(map[string]string)
	}

	for key, value := range labels {
		podTemplate.Labels[key] = value
	}

	// Generate TF_CONFIG JSON string.
	tfConfigStr := genTFConfigJSONStr(tfjob, rt, index)

	if tfConfigStr == "" {
		return nil
	}
	// Add TF_CONFIG environment variable.
	for i := range podTemplate.Spec.Containers {
		if len(podTemplate.Spec.Containers[i].Env) == 0 {
			podTemplate.Spec.Containers[i].Env = make([]v1.EnvVar, 0)
		}
		podTemplate.Spec.Containers[i].Env = append(podTemplate.Spec.Containers[i].Env, v1.EnvVar{
			Name:  "TF_CONFIG",
			Value: tfConfigStr,
		})
	}

	// TODO(gaocegege): Deal with RestartPolicyExitCode.
	// Set restart policy
	if spec.RestartPolicy != tfv1alpha2.RestartPolicyExitCode {
		podTemplate.Spec.RestartPolicy = v1.RestartPolicy(spec.RestartPolicy)
	}

	err = tc.podControl.CreatePodsWithControllerRef(tfjob.Namespace, podTemplate, tfjob, controllerRef)
	if err != nil && errors.IsTimeout(err) {
		// Pod is created but its initialization has timed out.
		// If the initialization is successful eventually, the
		// controller will observe the creation via the informer.
		// If the initialization fails, or if the pod keeps
		// uninitialized for a long time, the informer will not
		// receive any update, and the controller will create a new
		// pod when the expectation expires.
		return nil
	} else if err != nil {
		return err
	}
	return nil
}

// getPodsForTFJob returns the set of pods that this tfjob should manage.
// It also reconciles ControllerRef by adopting/orphaning.
// Note that the returned Pods are pointers into the cache.
func (tc *TFJobController) getPodsForTFJob(tfjob *tfv1alpha2.TFJob) ([]*v1.Pod, error) {
	tfjobKey, err := KeyFunc(tfjob)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("Couldn't get key for tfjob object %#v: %v", tfjob, err))
		return nil, err
	}

	// Create selector.
	selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
		MatchLabels: genLabels(tfjobKey),
	})

	if err != nil {
		return nil, fmt.Errorf("couldn't convert Job selector: %v", err)
	}
	// List all pods to include those that don't match the selector anymore
	// but have a ControllerRef pointing to this controller.
	pods, err := tc.podLister.Pods(tfjob.Namespace).List(labels.Everything())
	if err != nil {
		return nil, err
	}

	// If any adoptions are attempted, we should first recheck for deletion
	// with an uncached quorum read sometime after listing Pods (see #42639).
	canAdoptFunc := RecheckDeletionTimestamp(func() (metav1.Object, error) {
		fresh, err := tc.tfJobClientSet.KubeflowV1alpha2().TFJobs(tfjob.Namespace).Get(tfjob.Name, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}
		if fresh.UID != tfjob.UID {
			return nil, fmt.Errorf("original TFJob %v/%v is gone: got uid %v, wanted %v", tfjob.Namespace, tfjob.Name, fresh.UID, tfjob.UID)
		}
		return fresh, nil
	})
	cm := NewPodControllerRefManager(tc.podControl, tfjob, selector, controllerKind, canAdoptFunc)
	return cm.ClaimPods(pods)
}

// filterPodsForTFReplicaType returns pods belong to a TFReplicaType.
func filterPodsForTFReplicaType(pods []*v1.Pod, tfReplicaType string) []*v1.Pod {
	var result []*v1.Pod

	tfReplicaSelector := &metav1.LabelSelector{
		MatchLabels: make(map[string]string),
	}

	tfReplicaSelector.MatchLabels[tfReplicaTypeLabel] = tfReplicaType

	for _, pod := range pods {
		selector, _ := metav1.LabelSelectorAsSelector(tfReplicaSelector)
		if !selector.Matches(labels.Set(pod.Labels)) {
			continue
		}
		result = append(result, pod)
	}
	return result
}

func genExpectationPodsKey(tfjobKey, replicaType string) string {
	return tfjobKey + "/" + strings.ToLower(replicaType) + "/pods"
}

// When a pod is created, enqueue the tfjob that manages it and update its expectations.
func (tc *TFJobController) addPod(obj interface{}) {
	pod := obj.(*v1.Pod)
	if pod.DeletionTimestamp != nil {
		// on a restart of the controller controller, it's possible a new pod shows up in a state that
		// is already pending deletion. Prevent the pod from being a creation observation.
		// tc.deletePod(pod)
		return
	}

	// If it has a ControllerRef, that's all that matters.
	if controllerRef := metav1.GetControllerOf(pod); controllerRef != nil {
		tfjob := tc.resolveControllerRef(pod.Namespace, controllerRef)
		if tfjob == nil {
			log.Info("This pod's tfjob does not exists")
			return
		}

		tfjobKey, err := KeyFunc(tfjob)
		if err != nil {
			loggerForTFJob(tfjob).Infof("Failed to get the key of the tfjob: %v", err)
			return
		}

		if _, ok := pod.Labels[tfReplicaTypeLabel]; !ok {
			loggerForTFJob(tfjob).Info("This pod maybe not created by tf-operator")
			return
		}

		rtype := pod.Labels[tfReplicaTypeLabel]
		expectationPodsKey := genExpectationPodsKey(tfjobKey, rtype)

		tc.expectations.CreationObserved(expectationPodsKey)
		tc.enqueueTFJob(tfjob)

		return
	}

	// Otherwise, it's an orphan. Get a list of all matching controllers and sync
	// them to see if anyone wants to adopt it.
	// DO NOT observe creation because no controller should be waiting for an
	// orphan.
	// for _, tfjob := range tc.getPodJobs(pod) {
	// 	tc.enqueueTFJob(tfjob)
	// }
}

// When a pod is updated, figure out what tfjob/s manage it and wake them up.
// If the labels of the pod have changed we need to awaken both the old
// and new replica set. old and cur must be *v1.Pod types.
func (tc *TFJobController) updatePod(old, cur interface{}) {
	// TODO(CPH): handle this gracefully.
}

// When a pod is deleted, enqueue the tfjob that manages the pod and update its expectations.
// obj could be an *v1.Pod, or a DeletionFinalStateUnknown marker item.
func (tc *TFJobController) deletePod(obj interface{}) {
	// TODO(CPH): handle this gracefully.
}
<|endoftext|>


<commit_before>// +build linux

package console

import (
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/dotcloud/docker/pkg/label"
	"github.com/dotcloud/docker/pkg/system"
)

// Setup initializes the proper /dev/console inside the rootfs path
func Setup(rootfs, consolePath, mountLabel string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	stat, err := os.Stat(consolePath)
	if err != nil {
		return fmt.Errorf("stat console %s %s", consolePath, err)
	}
	var (
		st   = stat.Sys().(*syscall.Stat_t)
		dest = filepath.Join(rootfs, "dev/console")
	)
	if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("remove %s %s", dest, err)
	}
	if err := os.Chmod(consolePath, 0600); err != nil {
		return err
	}
	if err := os.Chown(consolePath, 0, 0); err != nil {
		return err
	}
	if err := system.Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil {
		return fmt.Errorf("mknod %s %s", dest, err)
	}
	if err := label.SetFileLabel(consolePath, mountLabel); err != nil {
		return fmt.Errorf("set file label %s %s", dest, err)
	}
	if err := system.Mount(consolePath, dest, "bind", syscall.MS_BIND, ""); err != nil {
		return fmt.Errorf("bind %s to %s %s", consolePath, dest, err)
	}
	return nil
}

func OpenAndDup(consolePath string) error {
	slave, err := system.OpenTerminal(consolePath, syscall.O_RDWR)
	if err != nil {
		return fmt.Errorf("open terminal %s", err)
	}
	if err := system.Dup2(slave.Fd(), 0); err != nil {
		return err
	}
	if err := system.Dup2(slave.Fd(), 1); err != nil {
		return err
	}
	return system.Dup2(slave.Fd(), 2)
}
<commit_msg>libcontainer: Don't create a device node on /dev/console to bind mount on<commit_after>// +build linux

package console

import (
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/dotcloud/docker/pkg/label"
	"github.com/dotcloud/docker/pkg/system"
)

// Setup initializes the proper /dev/console inside the rootfs path
func Setup(rootfs, consolePath, mountLabel string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	if err := os.Chmod(consolePath, 0600); err != nil {
		return err
	}
	if err := os.Chown(consolePath, 0, 0); err != nil {
		return err
	}
	if err := label.SetFileLabel(consolePath, mountLabel); err != nil {
		return fmt.Errorf("set file label %s %s", consolePath, err)
	}

	dest := filepath.Join(rootfs, "dev/console")

	f, err := os.Create(dest)
	if err != nil && !os.IsExist(err) {
		return fmt.Errorf("create %s %s", dest, err)
	}
	if f != nil {
		f.Close()
	}

	if err := system.Mount(consolePath, dest, "bind", syscall.MS_BIND, ""); err != nil {
		return fmt.Errorf("bind %s to %s %s", consolePath, dest, err)
	}
	return nil
}

func OpenAndDup(consolePath string) error {
	slave, err := system.OpenTerminal(consolePath, syscall.O_RDWR)
	if err != nil {
		return fmt.Errorf("open terminal %s", err)
	}
	if err := system.Dup2(slave.Fd(), 0); err != nil {
		return err
	}
	if err := system.Dup2(slave.Fd(), 1); err != nil {
		return err
	}
	return system.Dup2(slave.Fd(), 2)
}
<|endoftext|>