<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package codec

import (
	"bytes"
	"encoding/binary"
	"runtime"
	"unsafe"

	"github.com/juju/errors"
)

const (
	encGroupSize = 8
	encMarker    = byte(0xFF)
	encPad       = byte(0x0)
)

var (
	pads    = make([]byte, encGroupSize)
	encPads = []byte{encPad}
)

// EncodeBytes guarantees the encoded value is in ascending order for comparison,
// encoding with the following rule:
//  [group1][marker1]...[groupN][markerN]
//  group is 8 bytes slice which is padding with 0.
//  marker is `0xFF - padding 0 count`
// For example:
//   [] -> [0, 0, 0, 0, 0, 0, 0, 0, 247]
//   [1, 2, 3] -> [1, 2, 3, 0, 0, 0, 0, 0, 250]
//   [1, 2, 3, 0] -> [1, 2, 3, 0, 0, 0, 0, 0, 251]
//   [1, 2, 3, 4, 5, 6, 7, 8] -> [1, 2, 3, 4, 5, 6, 7, 8, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247]
// Refer: https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format
func EncodeBytes(b []byte, data []byte) []byte {
	// Allocate more space to avoid unnecessary slice growing.
	// Assume that the byte slice size is about `(len(data) / encGroupSize + 1) * (encGroupSize + 1)` bytes,
	// that is `(len(data) / 8 + 1) * 9` in our implement.
	dLen := len(data)
	reallocSize := (dLen/encGroupSize + 1) * (encGroupSize + 1)
	result := reallocBytes(b, reallocSize)
	for idx := 0; idx <= dLen; idx += encGroupSize {
		remain := dLen - idx
		padCount := 0
		if remain >= encGroupSize {
			result = append(result, data[idx:idx+encGroupSize]...)
		} else {
			padCount = encGroupSize - remain
			result = append(result, data[idx:]...)
			result = append(result, pads[:padCount]...)
		}

		marker := encMarker - byte(padCount)
		result = append(result, marker)
	}

	return result
}

func decodeBytes(b []byte, reverse bool) ([]byte, []byte, error) {
	data := make([]byte, 0, len(b))
	for {
		if len(b) < encGroupSize+1 {
			return nil, nil, errors.New("insufficient bytes to decode value")
		}

		groupBytes := b[:encGroupSize+1]
		if reverse {
			reverseBytes(groupBytes)
		}

		group := groupBytes[:encGroupSize]
		marker := groupBytes[encGroupSize]

		// Check validity of marker.
		padCount := encMarker - marker
		realGroupSize := encGroupSize - padCount
		if padCount > encGroupSize {
			return nil, nil, errors.Errorf("invalid marker byte, group bytes %q", groupBytes)
		}

		data = append(data, group[:realGroupSize]...)
		b = b[encGroupSize+1:]

		if marker != encMarker {
			// Check validity of padding bytes.
			if bytes.Count(group[realGroupSize:], encPads) != int(padCount) {
				return nil, nil, errors.Errorf("invalid padding byte, group bytes %q", groupBytes)
			}

			break
		}
	}

	return b, data, nil
}

// DecodeBytes decodes bytes which is encoded by EncodeBytes before,
// returns the leftover bytes and decoded value if no error.
func DecodeBytes(b []byte) ([]byte, []byte, error) {
	return decodeBytes(b, false)
}

// EncodeBytesDesc first encodes bytes using EncodeBytes, then bitwise reverses
// encoded value to guarantee the encoded value is in descending order for comparison.
func EncodeBytesDesc(b []byte, data []byte) []byte {
	n := len(b)
	b = EncodeBytes(b, data)
	reverseBytes(b[n:])
	return b
}

// DecodeBytesDesc decodes bytes which is encoded by EncodeBytesDesc before,
// returns the leftover bytes and decoded value if no error.
func DecodeBytesDesc(b []byte) ([]byte, []byte, error) {
	return decodeBytes(b, true)
}

// EncodeCompactBytes joins bytes with its length into a byte slice. It is more
// efficient in both space and time compare to EncodeBytes. Note that the encoded
// result is not memcomparable.
func EncodeCompactBytes(b []byte, data []byte) []byte {
	b = reallocBytes(b, binary.MaxVarintLen64+len(data))
	b = EncodeVarint(b, int64(len(data)))
	return append(b, data...)
}

// DecodeCompactBytes decodes bytes which is encoded by EncodeCompactBytes before.
func DecodeCompactBytes(b []byte) ([]byte, []byte, error) {
	b, n, err := DecodeVarint(b)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	if int64(len(b)) < n {
		return nil, nil, errors.Errorf("insufficient bytes to decode value, expected length: %v", n)
	}
	return b[n:], b[:n], nil
}

// See https://golang.org/src/crypto/cipher/xor.go
const wordSize = int(unsafe.Sizeof(uintptr(0)))
const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64"

func fastReverseBytes(b []byte) {
	n := len(b)
	w := n / wordSize
	if w > 0 {
		bw := *(*[]uintptr)(unsafe.Pointer(&b))
		for i := 0; i < w; i++ {
			bw[i] = ^bw[i]
		}
	}

	for i := w * wordSize; i < n; i++ {
		b[i] = ^b[i]
	}
}

func safeReverseBytes(b []byte) {
	for i := range b {
		b[i] = ^b[i]
	}
}

func reverseBytes(b []byte) {
	if supportsUnaligned {
		fastReverseBytes(b)
		return
	}

	safeReverseBytes(b)
}

// like realloc.
func reallocBytes(b []byte, n int) []byte {
	newSize := len(b) + n
	if cap(b) < newSize {
		bs := make([]byte, len(b), newSize)
		copy(bs, b)
		return bs
	}

	// slice b has capability to store n bytes
	return b
}
<commit_msg>util/codec: do not modify data while decoding bytes.<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package codec

import (
	"encoding/binary"
	"runtime"
	"unsafe"

	"github.com/juju/errors"
)

const (
	encGroupSize = 8
	encMarker    = byte(0xFF)
	encPad       = byte(0x0)
)

var (
	pads    = make([]byte, encGroupSize)
	encPads = []byte{encPad}
)

// EncodeBytes guarantees the encoded value is in ascending order for comparison,
// encoding with the following rule:
//  [group1][marker1]...[groupN][markerN]
//  group is 8 bytes slice which is padding with 0.
//  marker is `0xFF - padding 0 count`
// For example:
//   [] -> [0, 0, 0, 0, 0, 0, 0, 0, 247]
//   [1, 2, 3] -> [1, 2, 3, 0, 0, 0, 0, 0, 250]
//   [1, 2, 3, 0] -> [1, 2, 3, 0, 0, 0, 0, 0, 251]
//   [1, 2, 3, 4, 5, 6, 7, 8] -> [1, 2, 3, 4, 5, 6, 7, 8, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247]
// Refer: https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format
func EncodeBytes(b []byte, data []byte) []byte {
	// Allocate more space to avoid unnecessary slice growing.
	// Assume that the byte slice size is about `(len(data) / encGroupSize + 1) * (encGroupSize + 1)` bytes,
	// that is `(len(data) / 8 + 1) * 9` in our implement.
	dLen := len(data)
	reallocSize := (dLen/encGroupSize + 1) * (encGroupSize + 1)
	result := reallocBytes(b, reallocSize)
	for idx := 0; idx <= dLen; idx += encGroupSize {
		remain := dLen - idx
		padCount := 0
		if remain >= encGroupSize {
			result = append(result, data[idx:idx+encGroupSize]...)
		} else {
			padCount = encGroupSize - remain
			result = append(result, data[idx:]...)
			result = append(result, pads[:padCount]...)
		}

		marker := encMarker - byte(padCount)
		result = append(result, marker)
	}

	return result
}

func decodeBytes(b []byte, reverse bool) ([]byte, []byte, error) {
	data := make([]byte, 0, len(b))
	for {
		if len(b) < encGroupSize+1 {
			return nil, nil, errors.New("insufficient bytes to decode value")
		}

		groupBytes := b[:encGroupSize+1]

		group := groupBytes[:encGroupSize]
		marker := groupBytes[encGroupSize]

		var padCount byte
		if reverse {
			padCount = marker
		} else {
			padCount = encMarker - marker
		}
		if padCount > encGroupSize {
			return nil, nil, errors.Errorf("invalid marker byte, group bytes %q", groupBytes)
		}

		realGroupSize := encGroupSize - padCount
		data = append(data, group[:realGroupSize]...)
		b = b[encGroupSize+1:]

		if padCount != 0 {
			var padByte = encPad
			if reverse {
				padByte = encMarker
			}
			// Check validity of padding bytes.
			for _, v := range group[realGroupSize:] {
				if v != padByte {
					return nil, nil, errors.Errorf("invalid padding byte, group bytes %q", groupBytes)
				}
			}
			break
		}
	}
	if reverse {
		reverseBytes(data)
	}
	return b, data, nil
}

// DecodeBytes decodes bytes which is encoded by EncodeBytes before,
// returns the leftover bytes and decoded value if no error.
func DecodeBytes(b []byte) ([]byte, []byte, error) {
	return decodeBytes(b, false)
}

// EncodeBytesDesc first encodes bytes using EncodeBytes, then bitwise reverses
// encoded value to guarantee the encoded value is in descending order for comparison.
func EncodeBytesDesc(b []byte, data []byte) []byte {
	n := len(b)
	b = EncodeBytes(b, data)
	reverseBytes(b[n:])
	return b
}

// DecodeBytesDesc decodes bytes which is encoded by EncodeBytesDesc before,
// returns the leftover bytes and decoded value if no error.
func DecodeBytesDesc(b []byte) ([]byte, []byte, error) {
	return decodeBytes(b, true)
}

// EncodeCompactBytes joins bytes with its length into a byte slice. It is more
// efficient in both space and time compare to EncodeBytes. Note that the encoded
// result is not memcomparable.
func EncodeCompactBytes(b []byte, data []byte) []byte {
	b = reallocBytes(b, binary.MaxVarintLen64+len(data))
	b = EncodeVarint(b, int64(len(data)))
	return append(b, data...)
}

// DecodeCompactBytes decodes bytes which is encoded by EncodeCompactBytes before.
func DecodeCompactBytes(b []byte) ([]byte, []byte, error) {
	b, n, err := DecodeVarint(b)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	if int64(len(b)) < n {
		return nil, nil, errors.Errorf("insufficient bytes to decode value, expected length: %v", n)
	}
	return b[n:], b[:n], nil
}

// See https://golang.org/src/crypto/cipher/xor.go
const wordSize = int(unsafe.Sizeof(uintptr(0)))
const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64"

func fastReverseBytes(b []byte) {
	n := len(b)
	w := n / wordSize
	if w > 0 {
		bw := *(*[]uintptr)(unsafe.Pointer(&b))
		for i := 0; i < w; i++ {
			bw[i] = ^bw[i]
		}
	}

	for i := w * wordSize; i < n; i++ {
		b[i] = ^b[i]
	}
}

func safeReverseBytes(b []byte) {
	for i := range b {
		b[i] = ^b[i]
	}
}

func reverseBytes(b []byte) {
	if supportsUnaligned {
		fastReverseBytes(b)
		return
	}

	safeReverseBytes(b)
}

// like realloc.
func reallocBytes(b []byte, n int) []byte {
	newSize := len(b) + n
	if cap(b) < newSize {
		bs := make([]byte, len(b), newSize)
		copy(bs, b)
		return bs
	}

	// slice b has capability to store n bytes
	return b
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"database/sql"
	"net/url"

	_ "github.com/mattn/go-sqlite3"
	"go.uber.org/zap"

	"magneticod/bittorrent"

	"path"
	"os"
)

type engineType uint8

const (
	SQLITE engineType = 0
	POSTGRESQL = 1
)

type Database struct {
	database *sql.DB
	engine engineType
	newTorrents chan bittorrent.Metadata
}

// NewDatabase creates a new Database.
//
// url either starts with "sqlite:" or "postgresql:"
func NewDatabase(rawurl string) (*Database, error) {
	db := Database{}

	dbURL, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}

	switch dbURL.Scheme {
	case "sqlite":
		db.engine = SQLITE
		// All this pain is to make sure that an empty file exist (in case the database is not there
		// yet) so that sql.Open won't fail.
		dbDir, _ := path.Split(dbURL.Path)
		if err := os.MkdirAll(dbDir, 0755); err != nil {
			return nil, fmt.Errorf("for directory `%s`:  %s", dbDir, err.Error())
		}
		f, err := os.OpenFile(dbURL.Path, os.O_CREATE, 0666)
		if err != nil {
			return nil, fmt.Errorf("for file `%s`: %s", dbURL.Path, err.Error())
		}
		if err := f.Sync(); err != nil {
			return nil, fmt.Errorf("for file `%s`: %s", dbURL.Path, err.Error())
		}
		if err := f.Close(); err != nil {
			return nil, fmt.Errorf("for file `%s`: %s", dbURL.Path, err.Error())
		}
		db.database, err = sql.Open("sqlite3", dbURL.RawPath)

	case "postgresql":
		db.engine = POSTGRESQL
		db.database, err = sql.Open("postgresql", rawurl)

	default:
		return nil, fmt.Errorf("unknown URI scheme (or malformed URI)!")
	}

	// Check for errors from sql.Open()
	if err != nil {
		return nil, fmt.Errorf("sql.Open()!  %s", err.Error())
	}

	if err = db.database.Ping(); err != nil {
		return nil, fmt.Errorf("DB.Ping()!  %s", err.Error())
	}

	if err := db.setupDatabase(); err != nil {
		return nil, fmt.Errorf("setupDatabase()!  %s", err.Error())
	}

	db.newTorrents = make(chan bittorrent.Metadata, 10)

	return &db, nil
}

// AddNewTorrent adds a new torrent to the *queue* to be flushed to the persistent database.
func (db *Database) AddNewTorrent(torrent bittorrent.Metadata) error {
	for {
		select {
		case db.newTorrents <- torrent:
			return nil
		default:
			// newTorrents queue was full: flush and try again and again (and again)...
			err := db.flushNewTorrents()
			if err != nil {
				return err
			}
			continue
		}
	}
}

func (db *Database) flushNewTorrents() error {
	tx, err := db.database.Begin()
	if err != nil {
		return fmt.Errorf("sql.DB.Begin()!  %s", err.Error())
	}

	var nTorrents, nFiles uint
	for torrent := range db.newTorrents {
		res, err := tx.Exec("INSERT INTO torrents (info_hash, name, total_size, discovered_on) VALUES ($1, $2, $3, $4);",
			torrent.InfoHash, torrent.Name, torrent.TotalSize, torrent.DiscoveredOn)
		if err != nil {
			ourError := fmt.Errorf("error while INSERTing INTO torrents!  %s", err.Error())
			if err := tx.Rollback(); err != nil {
				return fmt.Errorf("%s\tmeanwhile, could not rollback the current transaction either!  %s", ourError.Error(), err.Error())
			}
			return ourError
		}
		var lastInsertId int64
		if lastInsertId, err = res.LastInsertId(); err != nil {
			return fmt.Errorf("sql.Result.LastInsertId()!  %s", err.Error())
		}

		for _, file := range torrent.Files {
			_, err := tx.Exec("INSERT INTO files (torrent_id, size, path) VALUES($1, $2, $3);",
			lastInsertId, file.Length, file.Path)
			if err != nil {
				ourError := fmt.Errorf("error while INSERTing INTO files!  %s", err.Error())
				if err := tx.Rollback(); err != nil {
					return fmt.Errorf("%s\tmeanwhile, could not rollback the current transaction either!  %s", ourError.Error(), err.Error())
				}
				return ourError
			}
			nFiles++
		}
		nTorrents++
	}

	err = tx.Commit()
	if err != nil {
		return fmt.Errorf("sql.Tx.Commit()!  %s", err.Error())
	}

	zap.L().Sugar().Infof("%d torrents (%d files) are flushed to the database successfully.",
		nTorrents, nFiles)
	return nil
}

func (db *Database) Close() {
	// Be careful to not to get into an infinite loop. =)
	db.database.Close()
}

func (db *Database) setupDatabase() error {
	switch db.engine {
	case SQLITE:
		return setupSqliteDatabase(db.database)

	case POSTGRESQL:
		zap.L().Fatal("setupDatabase() is not implemented for PostgreSQL yet!")

	default:
		zap.L().Sugar().Fatalf("Unknown database engine value %d! (programmer error)", db.engine)
	}

	return nil
}

func setupSqliteDatabase(database *sql.DB) error {
	// Enable Write-Ahead Logging for SQLite as "WAL provides more concurrency as readers do not
	// block writers and a writer does not block readers. Reading and writing can proceed
	// concurrently."
	// Caveats:
	//   * Might be unsupported by OSes other than Windows and UNIXes.
	//   * Does not work over a network filesystem.
	//   * Transactions that involve changes against multiple ATTACHed databases are not atomic
	//     across all databases as a set.
	// See: https://www.sqlite.org/wal.html
	//
	// Force SQLite to use disk, instead of memory, for all temporary files to reduce the memory
	// footprint.
	//
	// Enable foreign key constraints in SQLite which are crucial to prevent programmer errors on
	// our side.
	_, err := database.Exec(
		`PRAGMA journal_mode=WAL;
		PRAGMA temp_store=1;
		PRAGMA foreign_keys=ON;`,
	)
	if err != nil {
		return err
	}

	_, err = database.Exec(
		`CREATE TABLE IF NOT EXISTS torrents (
			id             INTEGER PRIMARY KEY,
			info_hash      BLOB NOT NULL UNIQUE,
			name           TEXT NOT NULL,
			total_size     INTEGER NOT NULL CHECK(total_size > 0),
			discovered_on  INTEGER NOT NULL CHECK(discovered_on > 0)
		);

		CREATE INDEX IF NOT EXISTS info_hash_index ON torrents (info_hash);

		CREATE TABLE IF NOT EXISTS files (
			id          INTEGER PRIMARY KEY,
			torrent_id  INTEGER REFERENCES torrents ON DELETE CASCADE ON UPDATE RESTRICT,
			size        INTEGER NOT NULL,
			path        TEXT NOT NULL
		);`,
	)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>magneticod: add mysql support<commit_after>package main

import (
	"fmt"
	"database/sql"
	"net/url"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/mattn/go-sqlite3"
	"go.uber.org/zap"

	"magneticod/bittorrent"

	"path"
	"os"
)

type engineType uint8

const (
	SQLITE     engineType = 0
	POSTGRESQL            = 1
	MYSQL                 = 2
)

type Database struct {
	database    *sql.DB
	engine      engineType
	newTorrents chan bittorrent.Metadata
}

// NewDatabase creates a new Database.
//
// url either starts with "sqlite:" or "postgresql:"
func NewDatabase(rawurl string) (*Database, error) {
	db := Database{}

	dbURL, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}

	switch dbURL.Scheme {
	case "sqlite":
		db.engine = SQLITE
		// All this pain is to make sure that an empty file exist (in case the database is not there
		// yet) so that sql.Open won't fail.
		dbDir, _ := path.Split(dbURL.Path)
		if err := os.MkdirAll(dbDir, 0755); err != nil {
			return nil, fmt.Errorf("for directory `%s`:  %s", dbDir, err.Error())
		}
		f, err := os.OpenFile(dbURL.Path, os.O_CREATE, 0666)
		if err != nil {
			return nil, fmt.Errorf("for file `%s`: %s", dbURL.Path, err.Error())
		}
		if err := f.Sync(); err != nil {
			return nil, fmt.Errorf("for file `%s`: %s", dbURL.Path, err.Error())
		}
		if err := f.Close(); err != nil {
			return nil, fmt.Errorf("for file `%s`: %s", dbURL.Path, err.Error())
		}
		db.database, err = sql.Open("sqlite3", dbURL.RawPath)

	case "postgresql":
		db.engine = POSTGRESQL
		db.database, err = sql.Open("postgresql", rawurl)

	case "mysql":
		db.engine = MYSQL
		db.database, err = sql.Open("mysql", rawurl)

	default:
		return nil, fmt.Errorf("unknown URI scheme (or malformed URI)!")
	}

	// Check for errors from sql.Open()
	if err != nil {
		return nil, fmt.Errorf("sql.Open()!  %s", err.Error())
	}

	if err = db.database.Ping(); err != nil {
		return nil, fmt.Errorf("DB.Ping()!  %s", err.Error())
	}

	if err := db.setupDatabase(); err != nil {
		return nil, fmt.Errorf("setupDatabase()!  %s", err.Error())
	}

	db.newTorrents = make(chan bittorrent.Metadata, 10)

	return &db, nil
}

// AddNewTorrent adds a new torrent to the *queue* to be flushed to the persistent database.
func (db *Database) AddNewTorrent(torrent bittorrent.Metadata) error {
	for {
		select {
		case db.newTorrents <- torrent:
			return nil
		default:
			// newTorrents queue was full: flush and try again and again (and again)...
			err := db.flushNewTorrents()
			if err != nil {
				return err
			}
			continue
		}
	}
}

func (db *Database) flushNewTorrents() error {
	tx, err := db.database.Begin()
	if err != nil {
		return fmt.Errorf("sql.DB.Begin()!  %s", err.Error())
	}

	var nTorrents, nFiles uint
	for torrent := range db.newTorrents {
		res, err := tx.Exec("INSERT INTO torrents (info_hash, name, total_size, discovered_on) VALUES ($1, $2, $3, $4);",
			torrent.InfoHash, torrent.Name, torrent.TotalSize, torrent.DiscoveredOn)
		if err != nil {
			ourError := fmt.Errorf("error while INSERTing INTO torrents!  %s", err.Error())
			if err := tx.Rollback(); err != nil {
				return fmt.Errorf("%s\tmeanwhile, could not rollback the current transaction either!  %s", ourError.Error(), err.Error())
			}
			return ourError
		}
		var lastInsertId int64
		if lastInsertId, err = res.LastInsertId(); err != nil {
			return fmt.Errorf("sql.Result.LastInsertId()!  %s", err.Error())
		}

		for _, file := range torrent.Files {
			_, err := tx.Exec("INSERT INTO files (torrent_id, size, path) VALUES($1, $2, $3);",
			lastInsertId, file.Length, file.Path)
			if err != nil {
				ourError := fmt.Errorf("error while INSERTing INTO files!  %s", err.Error())
				if err := tx.Rollback(); err != nil {
					return fmt.Errorf("%s\tmeanwhile, could not rollback the current transaction either!  %s", ourError.Error(), err.Error())
				}
				return ourError
			}
			nFiles++
		}
		nTorrents++
	}

	err = tx.Commit()
	if err != nil {
		return fmt.Errorf("sql.Tx.Commit()!  %s", err.Error())
	}

	zap.L().Sugar().Infof("%d torrents (%d files) are flushed to the database successfully.",
		nTorrents, nFiles)
	return nil
}

func (db *Database) Close() {
	// Be careful to not to get into an infinite loop. =)
	db.database.Close()
}

func (db *Database) setupDatabase() error {
	switch db.engine {
	case SQLITE:
		return setupSqliteDatabase(db.database)

	case POSTGRESQL:
		zap.L().Fatal("setupDatabase() is not implemented for PostgreSQL yet!")

	case MYSQL:
		return setupMySQLDatabase(db.database)

	default:
		zap.L().Sugar().Fatalf("Unknown database engine value %d! (programmer error)", db.engine)
	}

	return nil
}

func setupSqliteDatabase(database *sql.DB) error {
	// Enable Write-Ahead Logging for SQLite as "WAL provides more concurrency as readers do not
	// block writers and a writer does not block readers. Reading and writing can proceed
	// concurrently."
	// Caveats:
	//   * Might be unsupported by OSes other than Windows and UNIXes.
	//   * Does not work over a network filesystem.
	//   * Transactions that involve changes against multiple ATTACHed databases are not atomic
	//     across all databases as a set.
	// See: https://www.sqlite.org/wal.html
	//
	// Force SQLite to use disk, instead of memory, for all temporary files to reduce the memory
	// footprint.
	//
	// Enable foreign key constraints in SQLite which are crucial to prevent programmer errors on
	// our side.
	_, err := database.Exec(
		`PRAGMA journal_mode=WAL;
		PRAGMA temp_store=1;
		PRAGMA foreign_keys=ON;`,
	)
	if err != nil {
		return err
	}

	_, err = database.Exec(
		`CREATE TABLE IF NOT EXISTS torrents (
			id             INTEGER PRIMARY KEY,
			info_hash      BLOB NOT NULL UNIQUE,
			name           TEXT NOT NULL,
			total_size     INTEGER NOT NULL CHECK(total_size > 0),
			discovered_on  INTEGER NOT NULL CHECK(discovered_on > 0)
		);

		CREATE INDEX IF NOT EXISTS info_hash_index ON torrents (info_hash);

		CREATE TABLE IF NOT EXISTS files (
			id          INTEGER PRIMARY KEY,
			torrent_id  INTEGER REFERENCES torrents ON DELETE CASCADE ON UPDATE RESTRICT,
			size        INTEGER NOT NULL,
			path        TEXT NOT NULL
		);`,
	)
	if err != nil {
		return err
	}

	return nil
}

func setupMySQLDatabase(database *sql.DB) error {
	_, err := database.Exec(
		`CREATE TABLE IF NOT EXISTS torrents ("
			id		INTEGER PRIMARY KEY AUTO_INCREMENT,
			info_hash	BINARY(20) NOT NULL UNIQUE,
			name		VARCHAR(255) NOT NULL,
			total_size	BIGINT UNSIGNED NOT NULL,
			discovered_on	INTEGER UNSIGNED NOT NULL
		);

		ALTER TABLE torrents ADD INDEX info_hash_index (info_hash);

		CREATE TABLE IF NOT EXISTS files (
			id		INTEGER PRIMARY KEY AUTO_INCREMENT,
			torrent_id	INTEGER REFERENCES torrents (id) ON DELETE CASCADE ON UPDATE RESTRICT,
			size		BIGINT NOT NULL,
			path		TEXT NOT NULL
		);`,
	)

	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// +build linux

package firewall

import (
	"fmt"
	"github.com/flexiant/concerto/utils"
)

func driverName() string {
	return "iptables"
}

func apply(policy Policy) error {
	utils.RunCmd("iptables -F INPUT")
	utils.RunCmd("iptables -P INPUT DROP")
	utils.RunCmd("iptables -A INPUT -i lo -j ACCEPT")
	utils.RunCmd("iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT")

	for _, rule := range policy.Rules {
		utils.RunCmd(fmt.Sprintf("iptables -A INPUT -s %s -p %s --dport %d:%d -j ACCEPT", rule.Cidr, rule.Protocol, rule.MinPort, rule.MaxPort))
	}

	return nil
}

func flush() error {
	utils.RunCmd("iptables -P INPUT DROP")
	utils.RunCmd("iptables -F INPUT")
	return nil
}
<commit_msg>Inclusion of Iptables Path<commit_after>// +build linux

package firewall

import (
	"fmt"
	"github.com/flexiant/concerto/utils"
)

func driverName() string {
	return "iptables"
}

func apply(policy Policy) error {
	utils.RunCmd("/sbin/iptables -F INPUT")
	utils.RunCmd("/sbin/iptables -P INPUT DROP")
	utils.RunCmd("/sbin/iptables -A INPUT -i lo -j ACCEPT")
	utils.RunCmd("/sbin/iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT")

	for _, rule := range policy.Rules {
		utils.RunCmd(fmt.Sprintf("/sbin/iptables -A INPUT -s %s -p %s --dport %d:%d -j ACCEPT", rule.Cidr, rule.Protocol, rule.MinPort, rule.MaxPort))
	}

	return nil
}

func flush() error {
	utils.RunCmd("/sbin/iptables -P INPUT DROP")
	utils.RunCmd("/sbin/iptables -F INPUT")
	return nil
}
<|endoftext|>


<commit_before>package glfw

/*
#include "glfw/include/GLFW/glfw3.h"
#include "glfw/src/internal.h"

GLFWAPI VkResult glfwCreateWindowSurface(VkInstance instance, GLFWwindow* window, const VkAllocationCallbacks* allocator, VkSurfaceKHR* surface);

// Helper function for doing raw pointer arithmetic
static inline const char* getArrayIndex(const char** array, unsigned int index) {
	return array[index];
}

void* getVulkanProcAddr() {
	return vkGetInstanceProcAddr;
}
*/
import "C"
import (
	"errors"
	"reflect"
	"unsafe"
)

// VulkanSupported reports whether the Vulkan loader has been found. This check is performed by Init.
//
// The availability of a Vulkan loader does not by itself guarantee that window surface creation or
// even device creation is possible. Call GetRequiredInstanceExtensions to check whether the
// extensions necessary for Vulkan surface creation are available and GetPhysicalDevicePresentationSupport
// to check whether a queue family of a physical device supports image presentation.
func VulkanSupported() bool {
	return glfwbool(C.glfwVulkanSupported())
}

// GetVulkanGetInstanceProcAddress returns the function pointer used to find Vulkan core or
// extension functions. The return value of this function can be passed to the Vulkan library.
//
// Note that this function does not work the same way as the glfwGetInstanceProcAddress.
func GetVulkanGetInstanceProcAddress() unsafe.Pointer {
	return C.getVulkanProcAddr()
}

// GetRequiredInstanceExtensions returns a slice of Vulkan instance extension names required
// by GLFW for creating Vulkan surfaces for GLFW windows. If successful, the list will always
// contain VK_KHR_surface, so if you don't require any additional extensions you can pass this list
// directly to the VkInstanceCreateInfo struct.
//
// If Vulkan is not available on the machine, this function returns nil. Call
// VulkanSupported to check whether Vulkan is available.
//
// If Vulkan is available but no set of extensions allowing window surface creation was found, this
// function returns nil. You may still use Vulkan for off-screen rendering and compute work.
func (window *Window) GetRequiredInstanceExtensions() []string {
	var count C.uint32_t
	strarr := C.glfwGetRequiredInstanceExtensions(&count)
	if count == 0 {
		return nil
	}

	extensions := make([]string, count)
	for i := uint(0); i < uint(count); i++ {
		extensions[i] = C.GoString(C.getArrayIndex(strarr, C.uint(i)))
	}
	return extensions
}

// CreateWindowSurface creates a Vulkan surface for this window.
func (window *Window) CreateWindowSurface(instance interface{}, allocCallbacks unsafe.Pointer) (surface uintptr, err error) {
	if instance == nil {
		return 0, errors.New("vulkan: instance is nil")
	}
	val := reflect.ValueOf(instance)
	if val.Kind() != reflect.Ptr {
		return 0, errors.New("vulkan: instance is not a VkInstance (expected kind Ptr, got " + val.Kind().String() + ")")
	}
	var vulkanSurface C.VkSurfaceKHR
	ret := C.glfwCreateWindowSurface(
		(C.VkInstance)(unsafe.Pointer(reflect.ValueOf(instance).Pointer())), window.data,
		(*C.VkAllocationCallbacks)(allocCallbacks), (*C.VkSurfaceKHR)(unsafe.Pointer(&vulkanSurface)))
	if ret != C.VK_SUCCESS {
		return 0, errors.New("vulkan: error creating window surface")
	}
	return uintptr(unsafe.Pointer(&vulkanSurface)), nil
}
<commit_msg>glfwGetInstanceProcAddress reference added.<commit_after>package glfw

/*
#include "glfw/include/GLFW/glfw3.h"
#include "glfw/src/internal.h"

GLFWAPI VkResult glfwCreateWindowSurface(VkInstance instance, GLFWwindow* window, const VkAllocationCallbacks* allocator, VkSurfaceKHR* surface);
GLFWAPI GLFWvkproc glfwGetInstanceProcAddress(VkInstance instance, const char* procname);

// Helper function for doing raw pointer arithmetic
static inline const char* getArrayIndex(const char** array, unsigned int index) {
	return array[index];
}

void* getVulkanProcAddr() {
	return glfwGetInstanceProcAddress;
}
*/
import "C"
import (
	"errors"
	"reflect"
	"unsafe"
)

// VulkanSupported reports whether the Vulkan loader has been found. This check is performed by Init.
//
// The availability of a Vulkan loader does not by itself guarantee that window surface creation or
// even device creation is possible. Call GetRequiredInstanceExtensions to check whether the
// extensions necessary for Vulkan surface creation are available and GetPhysicalDevicePresentationSupport
// to check whether a queue family of a physical device supports image presentation.
func VulkanSupported() bool {
	return glfwbool(C.glfwVulkanSupported())
}

// GetVulkanGetInstanceProcAddress returns the function pointer used to find Vulkan core or
// extension functions. The return value of this function can be passed to the Vulkan library.
//
// Note that this function does not work the same way as the glfwGetInstanceProcAddress.
func GetVulkanGetInstanceProcAddress() unsafe.Pointer {
	return C.getVulkanProcAddr()
}

// GetRequiredInstanceExtensions returns a slice of Vulkan instance extension names required
// by GLFW for creating Vulkan surfaces for GLFW windows. If successful, the list will always
// contain VK_KHR_surface, so if you don't require any additional extensions you can pass this list
// directly to the VkInstanceCreateInfo struct.
//
// If Vulkan is not available on the machine, this function returns nil. Call
// VulkanSupported to check whether Vulkan is available.
//
// If Vulkan is available but no set of extensions allowing window surface creation was found, this
// function returns nil. You may still use Vulkan for off-screen rendering and compute work.
func (window *Window) GetRequiredInstanceExtensions() []string {
	var count C.uint32_t
	strarr := C.glfwGetRequiredInstanceExtensions(&count)
	if count == 0 {
		return nil
	}

	extensions := make([]string, count)
	for i := uint(0); i < uint(count); i++ {
		extensions[i] = C.GoString(C.getArrayIndex(strarr, C.uint(i)))
	}
	return extensions
}

// CreateWindowSurface creates a Vulkan surface for this window.
func (window *Window) CreateWindowSurface(instance interface{}, allocCallbacks unsafe.Pointer) (surface uintptr, err error) {
	if instance == nil {
		return 0, errors.New("vulkan: instance is nil")
	}
	val := reflect.ValueOf(instance)
	if val.Kind() != reflect.Ptr {
		return 0, errors.New("vulkan: instance is not a VkInstance (expected kind Ptr, got " + val.Kind().String() + ")")
	}
	var vulkanSurface C.VkSurfaceKHR
	ret := C.glfwCreateWindowSurface(
		(C.VkInstance)(unsafe.Pointer(reflect.ValueOf(instance).Pointer())), window.data,
		(*C.VkAllocationCallbacks)(allocCallbacks), (*C.VkSurfaceKHR)(unsafe.Pointer(&vulkanSurface)))
	if ret != C.VK_SUCCESS {
		return 0, errors.New("vulkan: error creating window surface")
	}
	return uintptr(unsafe.Pointer(&vulkanSurface)), nil
}
<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/mattbaird/elastigo/api"
)

var syncMappingCommand = cli.Command{
	Name:   "sync_mapping",
	Usage:  "sync the configuration definition with the store mappings",
	Action: doSyncMapping,
}

type mappingProto map[string]interface{}

type templateProto struct {
	Template string `json:"template"`
	Order    int    `json:"order"`
	Mappings map[string]mappingProto
}

func makeTemplate(pattern string, dynamicTemplates []mappingProto) map[string]interface{} {
	return mappingProto{
		"template": pattern + "*",
		"order":    1,
		"mappings": mappingProto{
			"_default_": mappingProto{
				"_timestamp": mappingProto{
					"enabled": true,
					"store":   true,
				},
				"dynamic_templates": dynamicTemplates,
			},
		},
	}
}

func notAnalyzedStringProto(pattern string) mappingProto {
	return mappingProto{
		pattern: mappingProto{
			"match":              pattern,
			"match_mapping_type": "string",
			"mapping": mappingProto{
				"index": "not_analyzed",
				"type":  "string",
			},
		},
	}
}

// doSyncMapping synchronizes the configuration definition with the Elastic
// Search backend mappings.
func doSyncMapping(c *cli.Context) {
	config := ParseConfigOrDie(c.GlobalString("config"))

	notAnalyzedProtos := []mappingProto{}
	for _, notAnalyzedPattern := range config.NotAnalyzedPatterns {
		notAnalyzedProtos = append(notAnalyzedProtos, notAnalyzedStringProto(notAnalyzedPattern))
	}

	for _, r := range config.Repositories {
		template := makeTemplate(r.IndexPrefix(), notAnalyzedProtos)
		if _, err := api.DoCommand("PUT", "/_template/vossibility", nil, template); err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Create one template per repository<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/mattbaird/elastigo/api"
)

var syncMappingCommand = cli.Command{
	Name:   "sync_mapping",
	Usage:  "sync the configuration definition with the store mappings",
	Action: doSyncMapping,
}

type mappingProto map[string]interface{}

type templateProto struct {
	Template string `json:"template"`
	Order    int    `json:"order"`
	Mappings map[string]mappingProto
}

func makeTemplate(pattern string, dynamicTemplates []mappingProto) map[string]interface{} {
	return mappingProto{
		"template": pattern + "*",
		"order":    1,
		"mappings": mappingProto{
			"_default_": mappingProto{
				"_timestamp": mappingProto{
					"enabled": true,
					"store":   true,
				},
				"dynamic_templates": dynamicTemplates,
			},
		},
	}
}

func notAnalyzedStringProto(pattern string) mappingProto {
	return mappingProto{
		pattern: mappingProto{
			"match":              pattern,
			"match_mapping_type": "string",
			"mapping": mappingProto{
				"index": "not_analyzed",
				"type":  "string",
			},
		},
	}
}

// doSyncMapping synchronizes the configuration definition with the Elastic
// Search backend mappings.
func doSyncMapping(c *cli.Context) {
	config := ParseConfigOrDie(c.GlobalString("config"))

	notAnalyzedProtos := []mappingProto{}
	for _, notAnalyzedPattern := range config.NotAnalyzedPatterns {
		notAnalyzedProtos = append(notAnalyzedProtos, notAnalyzedStringProto(notAnalyzedPattern))
	}

	for _, r := range config.Repositories {
		template := makeTemplate(r.IndexPrefix(), notAnalyzedProtos)
		if _, err := api.DoCommand("PUT", "/_template/vossibility-"+r.GivenName, nil, template); err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package olm

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/coreos/go-semver/semver"
	"github.com/operator-framework/api/pkg/operators/v1alpha1"
	olmErrors "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/errors"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
)

func (a *Operator) minKubeVersionStatus(name string, minKubeVersion string) (met bool, statuses []v1alpha1.RequirementStatus) {
	if minKubeVersion == "" {
		return true, nil
	}

	status := v1alpha1.RequirementStatus{
		Group:   "operators.coreos.com",
		Version: "v1alpha1",
		Kind:    "ClusterServiceVersion",
		Name:    name,
	}

	// Retrieve server k8s version
	serverVersionInfo, err := a.opClient.KubernetesInterface().Discovery().ServerVersion()
	if err != nil {
		status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
		status.Message = "Server version discovery error"
		met = false
		statuses = append(statuses, status)
		return
	}

	serverVersion, err := semver.NewVersion(strings.Split(strings.TrimPrefix(serverVersionInfo.String(), "v"), "-")[0])
	if err != nil {
		status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
		status.Message = "Server version parsing error"
		met = false
		statuses = append(statuses, status)
		return
	}

	csvVersionInfo, err := semver.NewVersion(strings.TrimPrefix(minKubeVersion, "v"))
	if err != nil {
		status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
		status.Message = "CSV version parsing error"
		met = false
		statuses = append(statuses, status)
		return
	}

	if csvVersionInfo.Compare(*serverVersion) > 0 {
		status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
		status.Message = fmt.Sprintf("CSV version requirement not met: minKubeVersion (%s) > server version (%s)", minKubeVersion, serverVersion.String())
		met = false
		statuses = append(statuses, status)
		return
	}

	status.Status = v1alpha1.RequirementStatusReasonPresent
	status.Message = fmt.Sprintf("CSV minKubeVersion (%s) less than server version (%s)", minKubeVersion, serverVersionInfo.String())
	met = true
	statuses = append(statuses, status)
	return
}

func (a *Operator) requirementStatus(strategyDetailsDeployment *v1alpha1.StrategyDetailsDeployment, crdDescs []v1alpha1.CRDDescription,
	ownedAPIServiceDescs []v1alpha1.APIServiceDescription, requiredAPIServiceDescs []v1alpha1.APIServiceDescription,
	requiredNativeAPIs []metav1.GroupVersionKind) (met bool, statuses []v1alpha1.RequirementStatus) {
	met = true

	// Check for CRDs
	for _, r := range crdDescs {
		status := v1alpha1.RequirementStatus{
			Group:   "apiextensions.k8s.io",
			Version: "v1",
			Kind:    "CustomResourceDefinition",
			Name:    r.Name,
		}

		// check if CRD exists - this verifies group, version, and kind, so no need for GVK check via discovery
		crd, err := a.lister.APIExtensionsV1().CustomResourceDefinitionLister().Get(r.Name)
		if err != nil {
			status.Status = v1alpha1.RequirementStatusReasonNotPresent
			status.Message = "CRD is not present"
			a.logger.Debugf("Setting 'met' to false, %v with status %v, with err: %v", r.Name, status, err)
			met = false
			statuses = append(statuses, status)
			continue
		}

		served := false
		for _, version := range crd.Spec.Versions {
			if version.Name == r.Version {
				if version.Served {
					served = true
				}
				break
			}
		}

		if !served {
			status.Status = v1alpha1.RequirementStatusReasonNotPresent
			status.Message = "CRD version not served"
			a.logger.Debugf("Setting 'met' to false, %v with status %v, CRD version %v not found", r.Name, status, r.Version)
			met = false
			statuses = append(statuses, status)
			continue
		}

		// Check if CRD has successfully registered with k8s API
		established := false
		namesAccepted := false
		for _, cdt := range crd.Status.Conditions {
			switch cdt.Type {
			case apiextensionsv1.Established:
				if cdt.Status == apiextensionsv1.ConditionTrue {
					established = true
				}
			case apiextensionsv1.NamesAccepted:
				if cdt.Status == apiextensionsv1.ConditionTrue {
					namesAccepted = true
				}
			}
		}

		if established && namesAccepted {
			status.Status = v1alpha1.RequirementStatusReasonPresent
			status.Message = "CRD is present and Established condition is true"
			status.UUID = string(crd.GetUID())
			statuses = append(statuses, status)
		} else {
			status.Status = v1alpha1.RequirementStatusReasonNotAvailable
			status.Message = "CRD is present but the Established condition is False (not available)"
			met = false
			a.logger.Debugf("Setting 'met' to false, %v with status %v, established=%v, namesAccepted=%v", r.Name, status, established, namesAccepted)
			statuses = append(statuses, status)
		}
	}

	// Check for required API services
	for _, r := range requiredAPIServiceDescs {
		name := fmt.Sprintf("%s.%s", r.Version, r.Group)
		status := v1alpha1.RequirementStatus{
			Group:   "apiregistration.k8s.io",
			Version: "v1",
			Kind:    "APIService",
			Name:    name,
		}

		// Check if GVK exists
		if err := a.isGVKRegistered(r.Group, r.Version, r.Kind); err != nil {
			status.Status = "NotPresent"
			met = false
			statuses = append(statuses, status)
			continue
		}

		// Check if APIService is registered
		apiService, err := a.lister.APIRegistrationV1().APIServiceLister().Get(name)
		if err != nil {
			status.Status = "NotPresent"
			met = false
			statuses = append(statuses, status)
			continue
		}

		// Check if API is available
		if !install.IsAPIServiceAvailable(apiService) {
			status.Status = "NotPresent"
			met = false
		} else {
			status.Status = "Present"
			status.UUID = string(apiService.GetUID())
		}
		statuses = append(statuses, status)
	}

	// Check owned API services
	for _, r := range ownedAPIServiceDescs {
		name := fmt.Sprintf("%s.%s", r.Version, r.Group)
		status := v1alpha1.RequirementStatus{
			Group:   "apiregistration.k8s.io",
			Version: "v1",
			Kind:    "APIService",
			Name:    name,
		}

		found := false
		for _, spec := range strategyDetailsDeployment.DeploymentSpecs {
			if spec.Name == r.DeploymentName {
				status.Status = "DeploymentFound"
				statuses = append(statuses, status)
				found = true
				break
			}
		}

		if !found {
			status.Status = "DeploymentNotFound"
			statuses = append(statuses, status)
			met = false
		}
	}

	for _, r := range requiredNativeAPIs {
		name := fmt.Sprintf("%s.%s", r.Version, r.Group)
		status := v1alpha1.RequirementStatus{
			Group:   r.Group,
			Version: r.Version,
			Kind:    r.Kind,
			Name:    name,
		}

		if err := a.isGVKRegistered(r.Group, r.Version, r.Kind); err != nil {
			status.Status = v1alpha1.RequirementStatusReasonNotPresent
			status.Message = "Native API does not exist"
			met = false
			statuses = append(statuses, status)
			continue
		} else {
			status.Status = v1alpha1.RequirementStatusReasonPresent
			status.Message = "Native API exists"
			statuses = append(statuses, status)
			continue
		}
	}

	return
}

// permissionStatus checks whether the given CSV's RBAC requirements are met in its namespace
func (a *Operator) permissionStatus(strategyDetailsDeployment *v1alpha1.StrategyDetailsDeployment, ruleChecker install.RuleChecker, targetNamespace string, csv *v1alpha1.ClusterServiceVersion) (bool, []v1alpha1.RequirementStatus, error) {
	statusesSet := map[string]v1alpha1.RequirementStatus{}

	checkPermissions := func(permissions []v1alpha1.StrategyDeploymentPermissions, namespace string) (bool, error) {
		met := true
		for _, perm := range permissions {
			saName := perm.ServiceAccountName
			a.logger.Debugf("perm.ServiceAccountName: %s", saName)

			var status v1alpha1.RequirementStatus
			if stored, ok := statusesSet[saName]; !ok {
				status = v1alpha1.RequirementStatus{
					Group:      "",
					Version:    "v1",
					Kind:       "ServiceAccount",
					Name:       saName,
					Status:     v1alpha1.RequirementStatusReasonPresent,
					Dependents: []v1alpha1.DependentStatus{},
				}
			} else {
				status = stored
			}

			// Ensure the ServiceAccount exists
			sa, err := a.opClient.GetServiceAccount(csv.GetNamespace(), perm.ServiceAccountName)
			if err != nil {
				met = false
				status.Status = v1alpha1.RequirementStatusReasonNotPresent
				status.Message = "Service account does not exist"
				statusesSet[saName] = status
				continue
			}

			// Check if the ServiceAccount is owned by CSV
			if !ownerutil.IsOwnedBy(sa, csv) {
				met = false
				status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
				status.Message = "Service account is not owned by this ClusterServiceVersion"
				statusesSet[saName] = status
				continue
			}

			// Check if PolicyRules are satisfied
			for _, rule := range perm.Rules {
				dependent := v1alpha1.DependentStatus{
					Group:   "rbac.authorization.k8s.io",
					Kind:    "PolicyRule",
					Version: "v1",
				}

				marshalled, err := json.Marshal(rule)
				if err != nil {
					dependent.Status = v1alpha1.DependentStatusReasonNotSatisfied
					dependent.Message = "rule unmarshallable"
					status.Dependents = append(status.Dependents, dependent)
					continue
				}

				var scope string
				if namespace == metav1.NamespaceAll {
					scope = "cluster"
				} else {
					scope = "namespaced"
				}
				dependent.Message = fmt.Sprintf("%s rule:%s", scope, marshalled)

				satisfied, err := ruleChecker.RuleSatisfied(sa, namespace, rule)
				if err != nil {
					return false, err
				} else if !satisfied {
					met = false
					dependent.Status = v1alpha1.DependentStatusReasonNotSatisfied
					status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
					status.Message = "Policy rule not satisfied for service account"
				} else {
					dependent.Status = v1alpha1.DependentStatusReasonSatisfied
				}

				status.Dependents = append(status.Dependents, dependent)
			}

			statusesSet[saName] = status
		}

		return met, nil
	}

	permMet, err := checkPermissions(strategyDetailsDeployment.Permissions, targetNamespace)
	if err != nil {
		return false, nil, err
	}
	clusterPermMet, err := checkPermissions(strategyDetailsDeployment.ClusterPermissions, metav1.NamespaceAll)
	if err != nil {
		return false, nil, err
	}

	statuses := []v1alpha1.RequirementStatus{}
	for key, status := range statusesSet {
		a.logger.WithField("key", key).WithField("status", status).Tracef("appending permission status")
		statuses = append(statuses, status)
	}

	return permMet && clusterPermMet, statuses, nil
}

// requirementAndPermissionStatus returns the aggregate requirement and permissions statuses for the given CSV
func (a *Operator) requirementAndPermissionStatus(csv *v1alpha1.ClusterServiceVersion) (bool, []v1alpha1.RequirementStatus, error) {
	allReqStatuses := []v1alpha1.RequirementStatus{}
	// Use a StrategyResolver to unmarshal
	strategyResolver := install.StrategyResolver{}
	strategy, err := strategyResolver.UnmarshalStrategy(csv.Spec.InstallStrategy)
	if err != nil {
		return false, nil, err
	}

	// Assume the strategy is for a deployment
	strategyDetailsDeployment, ok := strategy.(*v1alpha1.StrategyDetailsDeployment)
	if !ok {
		return false, nil, fmt.Errorf("could not cast install strategy as type %T", strategyDetailsDeployment)
	}

	// Check kubernetes version requirement between CSV and server
	minKubeMet, minKubeStatus := a.minKubeVersionStatus(csv.GetName(), csv.Spec.MinKubeVersion)
	if minKubeStatus != nil {
		allReqStatuses = append(allReqStatuses, minKubeStatus...)
	}

	reqMet, reqStatuses := a.requirementStatus(strategyDetailsDeployment, csv.GetAllCRDDescriptions(), csv.GetOwnedAPIServiceDescriptions(), csv.GetRequiredAPIServiceDescriptions(), csv.Spec.NativeAPIs)
	allReqStatuses = append(allReqStatuses, reqStatuses...)

	rbacLister := a.lister.RbacV1()
	roleLister := rbacLister.RoleLister()
	roleBindingLister := rbacLister.RoleBindingLister()
	clusterRoleLister := rbacLister.ClusterRoleLister()
	clusterRoleBindingLister := rbacLister.ClusterRoleBindingLister()

	ruleChecker := install.NewCSVRuleChecker(roleLister, roleBindingLister, clusterRoleLister, clusterRoleBindingLister, csv)
	permMet, permStatuses, err := a.permissionStatus(strategyDetailsDeployment, ruleChecker, csv.GetNamespace(), csv)
	if err != nil {
		return false, nil, err
	}

	// Aggregate requirement and permissions statuses
	statuses := append(allReqStatuses, permStatuses...)
	met := minKubeMet && reqMet && permMet
	if !met {
		a.logger.WithField("minKubeMet", minKubeMet).WithField("reqMet", reqMet).WithField("permMet", permMet).Debug("permissions/requirements not met")
	}

	return met, statuses, nil
}

func (a *Operator) isGVKRegistered(group, version, kind string) error {
	logger := a.logger.WithFields(logrus.Fields{
		"group":   group,
		"version": version,
		"kind":    kind,
	})

	gv := metav1.GroupVersion{Group: group, Version: version}
	resources, err := a.opClient.KubernetesInterface().Discovery().ServerResourcesForGroupVersion(gv.String())
	if err != nil {
		logger.WithField("err", err).Info("could not query for GVK in api discovery")
		return err
	}

	for _, r := range resources.APIResources {
		if r.Kind == kind {
			return nil
		}
	}

	logger.Info("couldn't find GVK in api discovery")
	return olmErrors.GroupVersionKindNotFoundError{group, version, kind}
}
<commit_msg>Only check the owner of the service account if the service account has owner<commit_after>package olm

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/coreos/go-semver/semver"
	"github.com/operator-framework/api/pkg/operators/v1alpha1"
	olmErrors "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/errors"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
)

func (a *Operator) minKubeVersionStatus(name string, minKubeVersion string) (met bool, statuses []v1alpha1.RequirementStatus) {
	if minKubeVersion == "" {
		return true, nil
	}

	status := v1alpha1.RequirementStatus{
		Group:   "operators.coreos.com",
		Version: "v1alpha1",
		Kind:    "ClusterServiceVersion",
		Name:    name,
	}

	// Retrieve server k8s version
	serverVersionInfo, err := a.opClient.KubernetesInterface().Discovery().ServerVersion()
	if err != nil {
		status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
		status.Message = "Server version discovery error"
		met = false
		statuses = append(statuses, status)
		return
	}

	serverVersion, err := semver.NewVersion(strings.Split(strings.TrimPrefix(serverVersionInfo.String(), "v"), "-")[0])
	if err != nil {
		status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
		status.Message = "Server version parsing error"
		met = false
		statuses = append(statuses, status)
		return
	}

	csvVersionInfo, err := semver.NewVersion(strings.TrimPrefix(minKubeVersion, "v"))
	if err != nil {
		status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
		status.Message = "CSV version parsing error"
		met = false
		statuses = append(statuses, status)
		return
	}

	if csvVersionInfo.Compare(*serverVersion) > 0 {
		status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
		status.Message = fmt.Sprintf("CSV version requirement not met: minKubeVersion (%s) > server version (%s)", minKubeVersion, serverVersion.String())
		met = false
		statuses = append(statuses, status)
		return
	}

	status.Status = v1alpha1.RequirementStatusReasonPresent
	status.Message = fmt.Sprintf("CSV minKubeVersion (%s) less than server version (%s)", minKubeVersion, serverVersionInfo.String())
	met = true
	statuses = append(statuses, status)
	return
}

func (a *Operator) requirementStatus(strategyDetailsDeployment *v1alpha1.StrategyDetailsDeployment, crdDescs []v1alpha1.CRDDescription,
	ownedAPIServiceDescs []v1alpha1.APIServiceDescription, requiredAPIServiceDescs []v1alpha1.APIServiceDescription,
	requiredNativeAPIs []metav1.GroupVersionKind) (met bool, statuses []v1alpha1.RequirementStatus) {
	met = true

	// Check for CRDs
	for _, r := range crdDescs {
		status := v1alpha1.RequirementStatus{
			Group:   "apiextensions.k8s.io",
			Version: "v1",
			Kind:    "CustomResourceDefinition",
			Name:    r.Name,
		}

		// check if CRD exists - this verifies group, version, and kind, so no need for GVK check via discovery
		crd, err := a.lister.APIExtensionsV1().CustomResourceDefinitionLister().Get(r.Name)
		if err != nil {
			status.Status = v1alpha1.RequirementStatusReasonNotPresent
			status.Message = "CRD is not present"
			a.logger.Debugf("Setting 'met' to false, %v with status %v, with err: %v", r.Name, status, err)
			met = false
			statuses = append(statuses, status)
			continue
		}

		served := false
		for _, version := range crd.Spec.Versions {
			if version.Name == r.Version {
				if version.Served {
					served = true
				}
				break
			}
		}

		if !served {
			status.Status = v1alpha1.RequirementStatusReasonNotPresent
			status.Message = "CRD version not served"
			a.logger.Debugf("Setting 'met' to false, %v with status %v, CRD version %v not found", r.Name, status, r.Version)
			met = false
			statuses = append(statuses, status)
			continue
		}

		// Check if CRD has successfully registered with k8s API
		established := false
		namesAccepted := false
		for _, cdt := range crd.Status.Conditions {
			switch cdt.Type {
			case apiextensionsv1.Established:
				if cdt.Status == apiextensionsv1.ConditionTrue {
					established = true
				}
			case apiextensionsv1.NamesAccepted:
				if cdt.Status == apiextensionsv1.ConditionTrue {
					namesAccepted = true
				}
			}
		}

		if established && namesAccepted {
			status.Status = v1alpha1.RequirementStatusReasonPresent
			status.Message = "CRD is present and Established condition is true"
			status.UUID = string(crd.GetUID())
			statuses = append(statuses, status)
		} else {
			status.Status = v1alpha1.RequirementStatusReasonNotAvailable
			status.Message = "CRD is present but the Established condition is False (not available)"
			met = false
			a.logger.Debugf("Setting 'met' to false, %v with status %v, established=%v, namesAccepted=%v", r.Name, status, established, namesAccepted)
			statuses = append(statuses, status)
		}
	}

	// Check for required API services
	for _, r := range requiredAPIServiceDescs {
		name := fmt.Sprintf("%s.%s", r.Version, r.Group)
		status := v1alpha1.RequirementStatus{
			Group:   "apiregistration.k8s.io",
			Version: "v1",
			Kind:    "APIService",
			Name:    name,
		}

		// Check if GVK exists
		if err := a.isGVKRegistered(r.Group, r.Version, r.Kind); err != nil {
			status.Status = "NotPresent"
			met = false
			statuses = append(statuses, status)
			continue
		}

		// Check if APIService is registered
		apiService, err := a.lister.APIRegistrationV1().APIServiceLister().Get(name)
		if err != nil {
			status.Status = "NotPresent"
			met = false
			statuses = append(statuses, status)
			continue
		}

		// Check if API is available
		if !install.IsAPIServiceAvailable(apiService) {
			status.Status = "NotPresent"
			met = false
		} else {
			status.Status = "Present"
			status.UUID = string(apiService.GetUID())
		}
		statuses = append(statuses, status)
	}

	// Check owned API services
	for _, r := range ownedAPIServiceDescs {
		name := fmt.Sprintf("%s.%s", r.Version, r.Group)
		status := v1alpha1.RequirementStatus{
			Group:   "apiregistration.k8s.io",
			Version: "v1",
			Kind:    "APIService",
			Name:    name,
		}

		found := false
		for _, spec := range strategyDetailsDeployment.DeploymentSpecs {
			if spec.Name == r.DeploymentName {
				status.Status = "DeploymentFound"
				statuses = append(statuses, status)
				found = true
				break
			}
		}

		if !found {
			status.Status = "DeploymentNotFound"
			statuses = append(statuses, status)
			met = false
		}
	}

	for _, r := range requiredNativeAPIs {
		name := fmt.Sprintf("%s.%s", r.Version, r.Group)
		status := v1alpha1.RequirementStatus{
			Group:   r.Group,
			Version: r.Version,
			Kind:    r.Kind,
			Name:    name,
		}

		if err := a.isGVKRegistered(r.Group, r.Version, r.Kind); err != nil {
			status.Status = v1alpha1.RequirementStatusReasonNotPresent
			status.Message = "Native API does not exist"
			met = false
			statuses = append(statuses, status)
			continue
		} else {
			status.Status = v1alpha1.RequirementStatusReasonPresent
			status.Message = "Native API exists"
			statuses = append(statuses, status)
			continue
		}
	}

	return
}

// permissionStatus checks whether the given CSV's RBAC requirements are met in its namespace
func (a *Operator) permissionStatus(strategyDetailsDeployment *v1alpha1.StrategyDetailsDeployment, ruleChecker install.RuleChecker, targetNamespace string, csv *v1alpha1.ClusterServiceVersion) (bool, []v1alpha1.RequirementStatus, error) {
	statusesSet := map[string]v1alpha1.RequirementStatus{}

	checkPermissions := func(permissions []v1alpha1.StrategyDeploymentPermissions, namespace string) (bool, error) {
		met := true
		for _, perm := range permissions {
			saName := perm.ServiceAccountName
			a.logger.Debugf("perm.ServiceAccountName: %s", saName)

			var status v1alpha1.RequirementStatus
			if stored, ok := statusesSet[saName]; !ok {
				status = v1alpha1.RequirementStatus{
					Group:      "",
					Version:    "v1",
					Kind:       "ServiceAccount",
					Name:       saName,
					Status:     v1alpha1.RequirementStatusReasonPresent,
					Dependents: []v1alpha1.DependentStatus{},
				}
			} else {
				status = stored
			}

			// Ensure the ServiceAccount exists
			sa, err := a.opClient.GetServiceAccount(csv.GetNamespace(), perm.ServiceAccountName)
			if err != nil {
				met = false
				status.Status = v1alpha1.RequirementStatusReasonNotPresent
				status.Message = "Service account does not exist"
				statusesSet[saName] = status
				continue
			}

			// Check if the ServiceAccount is owned by CSV
			if len(sa.GetOwnerReferences()) != 0 && !ownerutil.IsOwnedBy(sa, csv) {
				met = false
				status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
				status.Message = "Service account is not owned by this ClusterServiceVersion"
				statusesSet[saName] = status
				continue
			}

			// Check if PolicyRules are satisfied
			for _, rule := range perm.Rules {
				dependent := v1alpha1.DependentStatus{
					Group:   "rbac.authorization.k8s.io",
					Kind:    "PolicyRule",
					Version: "v1",
				}

				marshalled, err := json.Marshal(rule)
				if err != nil {
					dependent.Status = v1alpha1.DependentStatusReasonNotSatisfied
					dependent.Message = "rule unmarshallable"
					status.Dependents = append(status.Dependents, dependent)
					continue
				}

				var scope string
				if namespace == metav1.NamespaceAll {
					scope = "cluster"
				} else {
					scope = "namespaced"
				}
				dependent.Message = fmt.Sprintf("%s rule:%s", scope, marshalled)

				satisfied, err := ruleChecker.RuleSatisfied(sa, namespace, rule)
				if err != nil {
					return false, err
				} else if !satisfied {
					met = false
					dependent.Status = v1alpha1.DependentStatusReasonNotSatisfied
					status.Status = v1alpha1.RequirementStatusReasonPresentNotSatisfied
					status.Message = "Policy rule not satisfied for service account"
				} else {
					dependent.Status = v1alpha1.DependentStatusReasonSatisfied
				}

				status.Dependents = append(status.Dependents, dependent)
			}

			statusesSet[saName] = status
		}

		return met, nil
	}

	permMet, err := checkPermissions(strategyDetailsDeployment.Permissions, targetNamespace)
	if err != nil {
		return false, nil, err
	}
	clusterPermMet, err := checkPermissions(strategyDetailsDeployment.ClusterPermissions, metav1.NamespaceAll)
	if err != nil {
		return false, nil, err
	}

	statuses := []v1alpha1.RequirementStatus{}
	for key, status := range statusesSet {
		a.logger.WithField("key", key).WithField("status", status).Tracef("appending permission status")
		statuses = append(statuses, status)
	}

	return permMet && clusterPermMet, statuses, nil
}

// requirementAndPermissionStatus returns the aggregate requirement and permissions statuses for the given CSV
func (a *Operator) requirementAndPermissionStatus(csv *v1alpha1.ClusterServiceVersion) (bool, []v1alpha1.RequirementStatus, error) {
	allReqStatuses := []v1alpha1.RequirementStatus{}
	// Use a StrategyResolver to unmarshal
	strategyResolver := install.StrategyResolver{}
	strategy, err := strategyResolver.UnmarshalStrategy(csv.Spec.InstallStrategy)
	if err != nil {
		return false, nil, err
	}

	// Assume the strategy is for a deployment
	strategyDetailsDeployment, ok := strategy.(*v1alpha1.StrategyDetailsDeployment)
	if !ok {
		return false, nil, fmt.Errorf("could not cast install strategy as type %T", strategyDetailsDeployment)
	}

	// Check kubernetes version requirement between CSV and server
	minKubeMet, minKubeStatus := a.minKubeVersionStatus(csv.GetName(), csv.Spec.MinKubeVersion)
	if minKubeStatus != nil {
		allReqStatuses = append(allReqStatuses, minKubeStatus...)
	}

	reqMet, reqStatuses := a.requirementStatus(strategyDetailsDeployment, csv.GetAllCRDDescriptions(), csv.GetOwnedAPIServiceDescriptions(), csv.GetRequiredAPIServiceDescriptions(), csv.Spec.NativeAPIs)
	allReqStatuses = append(allReqStatuses, reqStatuses...)

	rbacLister := a.lister.RbacV1()
	roleLister := rbacLister.RoleLister()
	roleBindingLister := rbacLister.RoleBindingLister()
	clusterRoleLister := rbacLister.ClusterRoleLister()
	clusterRoleBindingLister := rbacLister.ClusterRoleBindingLister()

	ruleChecker := install.NewCSVRuleChecker(roleLister, roleBindingLister, clusterRoleLister, clusterRoleBindingLister, csv)
	permMet, permStatuses, err := a.permissionStatus(strategyDetailsDeployment, ruleChecker, csv.GetNamespace(), csv)
	if err != nil {
		return false, nil, err
	}

	// Aggregate requirement and permissions statuses
	statuses := append(allReqStatuses, permStatuses...)
	met := minKubeMet && reqMet && permMet
	if !met {
		a.logger.WithField("minKubeMet", minKubeMet).WithField("reqMet", reqMet).WithField("permMet", permMet).Debug("permissions/requirements not met")
	}

	return met, statuses, nil
}

func (a *Operator) isGVKRegistered(group, version, kind string) error {
	logger := a.logger.WithFields(logrus.Fields{
		"group":   group,
		"version": version,
		"kind":    kind,
	})

	gv := metav1.GroupVersion{Group: group, Version: version}
	resources, err := a.opClient.KubernetesInterface().Discovery().ServerResourcesForGroupVersion(gv.String())
	if err != nil {
		logger.WithField("err", err).Info("could not query for GVK in api discovery")
		return err
	}

	for _, r := range resources.APIResources {
		if r.Kind == kind {
			return nil
		}
	}

	logger.Info("couldn't find GVK in api discovery")
	return olmErrors.GroupVersionKindNotFoundError{group, version, kind}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nvidia

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"sync"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
	"k8s.io/kubernetes/pkg/kubelet/gpu"
)

// TODO: rework to use Nvidia's NVML, which is more complex, but also provides more fine-grained information and stats.
const (
	// All NVIDIA GPUs cards should be mounted with nvidiactl and nvidia-uvm
	// If the driver installed correctly, the 2 devices will be there.
	nvidiaCtlDevice string = "/dev/nvidiactl"
	nvidiaUVMDevice string = "/dev/nvidia-uvm"
	// Optional device.
	nvidiaUVMToolsDevice string = "/dev/nvidia-uvm-tools"
	devDirectory                = "/dev"
	nvidiaDeviceRE              = `^nvidia[0-9]*$`
	nvidiaFullpathRE            = `^/dev/nvidia[0-9]*$`
)

type activePodsLister interface {
	// Returns a list of active pods on the node.
	GetActivePods() []*v1.Pod
}

// nvidiaGPUManager manages nvidia gpu devices.
type nvidiaGPUManager struct {
	sync.Mutex
	// All gpus available on the Node
	allGPUs        sets.String
	allocated      *podGPUs
	defaultDevices []string
	// The interface which could get GPU mapping from all the containers.
	// TODO: Should make this independent of Docker in the future.
	dockerClient     libdocker.Interface
	activePodsLister activePodsLister
}

// NewNvidiaGPUManager returns a GPUManager that manages local Nvidia GPUs.
// TODO: Migrate to use pod level cgroups and make it generic to all runtimes.
func NewNvidiaGPUManager(activePodsLister activePodsLister, dockerClient libdocker.Interface) (gpu.GPUManager, error) {
	if dockerClient == nil {
		return nil, fmt.Errorf("invalid docker client specified")
	}
	return &nvidiaGPUManager{
		allGPUs:          sets.NewString(),
		dockerClient:     dockerClient,
		activePodsLister: activePodsLister,
	}, nil
}

// Initialize the GPU devices, so far only needed to discover the GPU paths.
func (ngm *nvidiaGPUManager) Start() error {
	if ngm.dockerClient == nil {
		return fmt.Errorf("Invalid docker client specified in GPU Manager")
	}
	ngm.Lock()
	defer ngm.Unlock()

	if _, err := os.Stat(nvidiaCtlDevice); err != nil {
		return err
	}

	if _, err := os.Stat(nvidiaUVMDevice); err != nil {
		return err
	}
	ngm.defaultDevices = []string{nvidiaCtlDevice, nvidiaUVMDevice}
	_, err := os.Stat(nvidiaUVMToolsDevice)
	if !os.IsNotExist(err) {
		ngm.defaultDevices = append(ngm.defaultDevices, nvidiaUVMToolsDevice)
	}

	if err := ngm.discoverGPUs(); err != nil {
		return err
	}
	// It's possible that the runtime isn't available now.
	ngm.allocated = ngm.gpusInUse()
	// We ignore errors when identifying allocated GPUs because it is possible that the runtime interfaces may be not be logically up.
	return nil
}

// Get how many GPU cards we have.
func (ngm *nvidiaGPUManager) Capacity() v1.ResourceList {
	gpus := resource.NewQuantity(int64(len(ngm.allGPUs)), resource.DecimalSI)
	return v1.ResourceList{
		v1.ResourceNvidiaGPU: *gpus,
	}
}

// AllocateGPUs returns `num` GPUs if available, error otherwise.
// Allocation is made thread safe using the following logic.
// A list of all GPUs allocated is maintained along with their respective Pod UIDs.
// It is expected that the list of active pods will not return any false positives.
// As part of initialization or allocation, the list of GPUs in use will be computed once.
// Whenever an allocation happens, the list of GPUs allocated is updated based on the list of currently active pods.
// GPUs allocated to terminated pods are freed up lazily as part of allocation.
// GPUs are allocated based on the internal list of allocatedGPUs.
// It is not safe to generate a list of GPUs in use by inspecting active containers because of the delay between GPU allocation and container creation.
// A GPU allocated to a container might be re-allocated to a subsequent container because the original container wasn't started quick enough.
// The current algorithm scans containers only once and then uses a list of active pods to track GPU usage.
// This is a sub-optimal solution and a better alternative would be that of using pod level cgroups instead.
// GPUs allocated to containers should be reflected in pod level device cgroups before completing allocations.
// The pod level cgroups will then serve as a checkpoint of GPUs in use.
func (ngm *nvidiaGPUManager) AllocateGPU(pod *v1.Pod, container *v1.Container) ([]string, error) {
	gpusNeeded := container.Resources.Limits.NvidiaGPU().Value()
	if gpusNeeded == 0 {
		return []string{}, nil
	}
	ngm.Lock()
	defer ngm.Unlock()
	if ngm.allocated == nil {
		// Initialization is not complete. Try now. Failures can no longer be tolerated.
		ngm.allocated = ngm.gpusInUse()
	} else {
		// update internal list of GPUs in use prior to allocating new GPUs.
		ngm.updateAllocatedGPUs()
	}
	// Check if GPUs have already been allocated. If so return them right away.
	// This can happen if a container restarts for example.
	if devices := ngm.allocated.getGPUs(string(pod.UID), container.Name); devices != nil {
		glog.V(2).Infof("Found pre-allocated GPUs for container %q in Pod %q: %v", container.Name, pod.UID, devices.List())
		return append(devices.List(), ngm.defaultDevices...), nil
	}
	// Get GPU devices in use.
	devicesInUse := ngm.allocated.devices()
	glog.V(5).Infof("gpus in use: %v", devicesInUse.List())
	// Get a list of available GPUs.
	available := ngm.allGPUs.Difference(devicesInUse)
	glog.V(5).Infof("gpus available: %v", available.List())
	if int64(available.Len()) < gpusNeeded {
		return nil, fmt.Errorf("requested number of GPUs unavailable. Requested: %d, Available: %d", gpusNeeded, available.Len())
	}
	ret := available.UnsortedList()[:gpusNeeded]
	for _, device := range ret {
		// Update internal allocated GPU cache.
		ngm.allocated.insert(string(pod.UID), container.Name, device)
	}
	// Add standard devices files that needs to be exposed.
	ret = append(ret, ngm.defaultDevices...)

	return ret, nil
}

// updateAllocatedGPUs updates the list of GPUs in use.
// It gets a list of active pods and then frees any GPUs that are bound to terminated pods.
// Returns error on failure.
func (ngm *nvidiaGPUManager) updateAllocatedGPUs() {
	activePods := ngm.activePodsLister.GetActivePods()
	activePodUids := sets.NewString()
	for _, pod := range activePods {
		activePodUids.Insert(string(pod.UID))
	}
	allocatedPodUids := ngm.allocated.pods()
	podsToBeRemoved := allocatedPodUids.Difference(activePodUids)
	glog.V(5).Infof("pods to be removed: %v", podsToBeRemoved.List())
	ngm.allocated.delete(podsToBeRemoved.List())
}

// discoverGPUs identifies allGPUs NVIDIA GPU devices available on the local node by walking `/dev` directory.
// TODO: Without NVML support we only can check whether there has GPU devices, but
// could not give a health check or get more information like GPU cores, memory, or
// family name. Need to support NVML in the future. But we do not need NVML until
// we want more features, features like schedule containers according to GPU family
// name.
func (ngm *nvidiaGPUManager) discoverGPUs() error {
	reg := regexp.MustCompile(nvidiaDeviceRE)
	files, err := ioutil.ReadDir(devDirectory)
	if err != nil {
		return err
	}
	for _, f := range files {
		if f.IsDir() {
			continue
		}
		if reg.MatchString(f.Name()) {
			glog.V(2).Infof("Found Nvidia GPU %q", f.Name())
			ngm.allGPUs.Insert(path.Join(devDirectory, f.Name()))
		}
	}

	return nil
}

// gpusInUse returns a list of GPUs in use along with the respective pods that are using it.
func (ngm *nvidiaGPUManager) gpusInUse() *podGPUs {
	pods := ngm.activePodsLister.GetActivePods()
	type containerIdentifier struct {
		id   string
		name string
	}
	type podContainers struct {
		uid        string
		containers []containerIdentifier
	}
	// List of containers to inspect.
	podContainersToInspect := []podContainers{}
	for _, pod := range pods {
		containers := sets.NewString()
		for _, container := range pod.Spec.Containers {
			// GPUs are expected to be specified only in limits.
			if !container.Resources.Limits.NvidiaGPU().IsZero() {
				containers.Insert(container.Name)
			}
		}
		// If no GPUs were requested skip this pod.
		if containers.Len() == 0 {
			continue
		}
		// TODO: If kubelet restarts right after allocating a GPU to a pod, the container might not have started yet and so container status might not be available yet.
		// Use an internal checkpoint instead or try using the CRI if its checkpoint is reliable.
		var containersToInspect []containerIdentifier
		for _, container := range pod.Status.ContainerStatuses {
			if containers.Has(container.Name) {
				containersToInspect = append(containersToInspect, containerIdentifier{container.ContainerID, container.Name})
			}
		}
		// add the pod and its containers that need to be inspected.
		podContainersToInspect = append(podContainersToInspect, podContainers{string(pod.UID), containersToInspect})
	}
	ret := newPodGPUs()
	for _, podContainer := range podContainersToInspect {
		for _, containerIdentifier := range podContainer.containers {
			containerJSON, err := ngm.dockerClient.InspectContainer(containerIdentifier.id)
			if err != nil {
				glog.V(3).Infof("Failed to inspect container %q in pod %q while attempting to reconcile nvidia gpus in use", containerIdentifier.id, podContainer.uid)
				continue
			}

			devices := containerJSON.HostConfig.Devices
			if devices == nil {
				continue
			}

			for _, device := range devices {
				if isValidPath(device.PathOnHost) {
					glog.V(4).Infof("Nvidia GPU %q is in use by Docker Container: %q", device.PathOnHost, containerJSON.ID)
					ret.insert(podContainer.uid, containerIdentifier.name, device.PathOnHost)
				}
			}
		}
	}
	return ret
}

func isValidPath(path string) bool {
	return regexp.MustCompile(nvidiaFullpathRE).MatchString(path)
}
<commit_msg>gpusInUse info error when kubelet restarts<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nvidia

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"
	"sync"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
	"k8s.io/kubernetes/pkg/kubelet/gpu"
)

// TODO: rework to use Nvidia's NVML, which is more complex, but also provides more fine-grained information and stats.
const (
	// All NVIDIA GPUs cards should be mounted with nvidiactl and nvidia-uvm
	// If the driver installed correctly, the 2 devices will be there.
	nvidiaCtlDevice string = "/dev/nvidiactl"
	nvidiaUVMDevice string = "/dev/nvidia-uvm"
	// Optional device.
	nvidiaUVMToolsDevice string = "/dev/nvidia-uvm-tools"
	devDirectory                = "/dev"
	nvidiaDeviceRE              = `^nvidia[0-9]*$`
	nvidiaFullpathRE            = `^/dev/nvidia[0-9]*$`
)

type activePodsLister interface {
	// Returns a list of active pods on the node.
	GetActivePods() []*v1.Pod
}

// nvidiaGPUManager manages nvidia gpu devices.
type nvidiaGPUManager struct {
	sync.Mutex
	// All gpus available on the Node
	allGPUs        sets.String
	allocated      *podGPUs
	defaultDevices []string
	// The interface which could get GPU mapping from all the containers.
	// TODO: Should make this independent of Docker in the future.
	dockerClient     libdocker.Interface
	activePodsLister activePodsLister
}

// NewNvidiaGPUManager returns a GPUManager that manages local Nvidia GPUs.
// TODO: Migrate to use pod level cgroups and make it generic to all runtimes.
func NewNvidiaGPUManager(activePodsLister activePodsLister, dockerClient libdocker.Interface) (gpu.GPUManager, error) {
	if dockerClient == nil {
		return nil, fmt.Errorf("invalid docker client specified")
	}
	return &nvidiaGPUManager{
		allGPUs:          sets.NewString(),
		dockerClient:     dockerClient,
		activePodsLister: activePodsLister,
	}, nil
}

// Initialize the GPU devices, so far only needed to discover the GPU paths.
func (ngm *nvidiaGPUManager) Start() error {
	if ngm.dockerClient == nil {
		return fmt.Errorf("Invalid docker client specified in GPU Manager")
	}
	ngm.Lock()
	defer ngm.Unlock()

	if _, err := os.Stat(nvidiaCtlDevice); err != nil {
		return err
	}

	if _, err := os.Stat(nvidiaUVMDevice); err != nil {
		return err
	}
	ngm.defaultDevices = []string{nvidiaCtlDevice, nvidiaUVMDevice}
	_, err := os.Stat(nvidiaUVMToolsDevice)
	if !os.IsNotExist(err) {
		ngm.defaultDevices = append(ngm.defaultDevices, nvidiaUVMToolsDevice)
	}

	if err := ngm.discoverGPUs(); err != nil {
		return err
	}

	// We ignore errors when identifying allocated GPUs because it is possible that the runtime interfaces may be not be logically up.
	return nil
}

// Get how many GPU cards we have.
func (ngm *nvidiaGPUManager) Capacity() v1.ResourceList {
	gpus := resource.NewQuantity(int64(len(ngm.allGPUs)), resource.DecimalSI)
	return v1.ResourceList{
		v1.ResourceNvidiaGPU: *gpus,
	}
}

// AllocateGPUs returns `num` GPUs if available, error otherwise.
// Allocation is made thread safe using the following logic.
// A list of all GPUs allocated is maintained along with their respective Pod UIDs.
// It is expected that the list of active pods will not return any false positives.
// As part of initialization or allocation, the list of GPUs in use will be computed once.
// Whenever an allocation happens, the list of GPUs allocated is updated based on the list of currently active pods.
// GPUs allocated to terminated pods are freed up lazily as part of allocation.
// GPUs are allocated based on the internal list of allocatedGPUs.
// It is not safe to generate a list of GPUs in use by inspecting active containers because of the delay between GPU allocation and container creation.
// A GPU allocated to a container might be re-allocated to a subsequent container because the original container wasn't started quick enough.
// The current algorithm scans containers only once and then uses a list of active pods to track GPU usage.
// This is a sub-optimal solution and a better alternative would be that of using pod level cgroups instead.
// GPUs allocated to containers should be reflected in pod level device cgroups before completing allocations.
// The pod level cgroups will then serve as a checkpoint of GPUs in use.
func (ngm *nvidiaGPUManager) AllocateGPU(pod *v1.Pod, container *v1.Container) ([]string, error) {
	gpusNeeded := container.Resources.Limits.NvidiaGPU().Value()
	if gpusNeeded == 0 {
		return []string{}, nil
	}
	ngm.Lock()
	defer ngm.Unlock()
	if ngm.allocated == nil {
		// Initialization is not complete. Try now. Failures can no longer be tolerated.
		ngm.allocated = ngm.gpusInUse()
	} else {
		// update internal list of GPUs in use prior to allocating new GPUs.
		ngm.updateAllocatedGPUs()
	}
	// Check if GPUs have already been allocated. If so return them right away.
	// This can happen if a container restarts for example.
	if devices := ngm.allocated.getGPUs(string(pod.UID), container.Name); devices != nil {
		glog.V(2).Infof("Found pre-allocated GPUs for container %q in Pod %q: %v", container.Name, pod.UID, devices.List())
		return append(devices.List(), ngm.defaultDevices...), nil
	}
	// Get GPU devices in use.
	devicesInUse := ngm.allocated.devices()
	glog.V(5).Infof("gpus in use: %v", devicesInUse.List())
	// Get a list of available GPUs.
	available := ngm.allGPUs.Difference(devicesInUse)
	glog.V(5).Infof("gpus available: %v", available.List())
	if int64(available.Len()) < gpusNeeded {
		return nil, fmt.Errorf("requested number of GPUs unavailable. Requested: %d, Available: %d", gpusNeeded, available.Len())
	}
	ret := available.UnsortedList()[:gpusNeeded]
	for _, device := range ret {
		// Update internal allocated GPU cache.
		ngm.allocated.insert(string(pod.UID), container.Name, device)
	}
	// Add standard devices files that needs to be exposed.
	ret = append(ret, ngm.defaultDevices...)

	return ret, nil
}

// updateAllocatedGPUs updates the list of GPUs in use.
// It gets a list of active pods and then frees any GPUs that are bound to terminated pods.
// Returns error on failure.
func (ngm *nvidiaGPUManager) updateAllocatedGPUs() {
	activePods := ngm.activePodsLister.GetActivePods()
	activePodUids := sets.NewString()
	for _, pod := range activePods {
		activePodUids.Insert(string(pod.UID))
	}
	allocatedPodUids := ngm.allocated.pods()
	podsToBeRemoved := allocatedPodUids.Difference(activePodUids)
	glog.V(5).Infof("pods to be removed: %v", podsToBeRemoved.List())
	ngm.allocated.delete(podsToBeRemoved.List())
}

// discoverGPUs identifies allGPUs NVIDIA GPU devices available on the local node by walking `/dev` directory.
// TODO: Without NVML support we only can check whether there has GPU devices, but
// could not give a health check or get more information like GPU cores, memory, or
// family name. Need to support NVML in the future. But we do not need NVML until
// we want more features, features like schedule containers according to GPU family
// name.
func (ngm *nvidiaGPUManager) discoverGPUs() error {
	reg := regexp.MustCompile(nvidiaDeviceRE)
	files, err := ioutil.ReadDir(devDirectory)
	if err != nil {
		return err
	}
	for _, f := range files {
		if f.IsDir() {
			continue
		}
		if reg.MatchString(f.Name()) {
			glog.V(2).Infof("Found Nvidia GPU %q", f.Name())
			ngm.allGPUs.Insert(path.Join(devDirectory, f.Name()))
		}
	}

	return nil
}

// gpusInUse returns a list of GPUs in use along with the respective pods that are using it.
func (ngm *nvidiaGPUManager) gpusInUse() *podGPUs {
	pods := ngm.activePodsLister.GetActivePods()
	type containerIdentifier struct {
		id   string
		name string
	}
	type podContainers struct {
		uid        string
		containers []containerIdentifier
	}
	// List of containers to inspect.
	podContainersToInspect := []podContainers{}
	for _, pod := range pods {
		containers := sets.NewString()
		for _, container := range pod.Spec.Containers {
			// GPUs are expected to be specified only in limits.
			if !container.Resources.Limits.NvidiaGPU().IsZero() {
				containers.Insert(container.Name)
			}
		}
		// If no GPUs were requested skip this pod.
		if containers.Len() == 0 {
			continue
		}
		// TODO: If kubelet restarts right after allocating a GPU to a pod, the container might not have started yet and so container status might not be available yet.
		// Use an internal checkpoint instead or try using the CRI if its checkpoint is reliable.
		var containersToInspect []containerIdentifier
		for _, container := range pod.Status.ContainerStatuses {
			if containers.Has(container.Name) {
				containersToInspect = append(containersToInspect, containerIdentifier{strings.Replace(container.ContainerID, "docker://", "", 1), container.Name})
			}
		}
		// add the pod and its containers that need to be inspected.
		podContainersToInspect = append(podContainersToInspect, podContainers{string(pod.UID), containersToInspect})
	}
	ret := newPodGPUs()
	for _, podContainer := range podContainersToInspect {
		for _, containerIdentifier := range podContainer.containers {
			containerJSON, err := ngm.dockerClient.InspectContainer(containerIdentifier.id)
			if err != nil {
				glog.V(3).Infof("Failed to inspect container %q in pod %q while attempting to reconcile nvidia gpus in use", containerIdentifier.id, podContainer.uid)
				continue
			}

			devices := containerJSON.HostConfig.Devices
			if devices == nil {
				continue
			}

			for _, device := range devices {
				if isValidPath(device.PathOnHost) {
					glog.V(4).Infof("Nvidia GPU %q is in use by Docker Container: %q", device.PathOnHost, containerJSON.ID)
					ret.insert(podContainer.uid, containerIdentifier.name, device.PathOnHost)
				}
			}
		}
	}
	return ret
}

func isValidPath(path string) bool {
	return regexp.MustCompile(nvidiaFullpathRE).MatchString(path)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package allocator

import (
	"math/big"
	"math/bits"
)

// countBits returns the number of set bits in n
func countBits(n *big.Int) int {
	var count int = 0
	for _, b := range n.Bytes() {
		count += bits.OnesCount8(uint8(b))
	}
	return count
}
<commit_msg>Speed up counting of bits in allocator<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package allocator

import (
	"math/big"
	"math/bits"
)

// countBits returns the number of set bits in n
func countBits(n *big.Int) int {
	var count int = 0
	for _, w := range n.Bits() {
		count += bits.OnesCount64(uint64(w))
	}
	return count
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package connections

import (
	"crypto/tls"
	"net"
	"net/url"
	"strings"

	"github.com/syncthing/syncthing/lib/dialer"
	"github.com/syncthing/syncthing/lib/model"
)

func init() {
	dialers["tcp"] = tcpDialer
	listeners["tcp"] = tcpListener
}

func tcpDialer(uri *url.URL, tlsCfg *tls.Config) (*tls.Conn, error) {
	// Check that there is a port number in uri.Host, otherwise add one.
	host, port, err := net.SplitHostPort(uri.Host)
	if err != nil && strings.HasPrefix(err.Error(), "missing port") {
		// addr is on the form "1.2.3.4"
		uri.Host = net.JoinHostPort(uri.Host, "22000")
	} else if err == nil && port == "" {
		// addr is on the form "1.2.3.4:"
		uri.Host = net.JoinHostPort(host, "22000")
	}

	// Don't try to resolve the address before dialing. The dialer may be a
	// proxy, and we should let the proxy do the resolving in that case.
	conn, err := dialer.Dial("tcp", uri.Host)
	if err != nil {
		l.Debugln(err)
		return nil, err
	}

	tc := tls.Client(conn, tlsCfg)
	err = tc.Handshake()
	if err != nil {
		tc.Close()
		return nil, err
	}

	return tc, nil
}

func tcpListener(uri *url.URL, tlsCfg *tls.Config, conns chan<- model.IntermediateConnection) {
	tcaddr, err := net.ResolveTCPAddr("tcp", uri.Host)
	if err != nil {
		l.Fatalln("listen (BEP/tcp):", err)
		return
	}
	listener, err := net.ListenTCP("tcp", tcaddr)
	if err != nil {
		l.Fatalln("listen (BEP/tcp):", err)
		return
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			l.Warnln("Accepting connection (BEP/tcp):", err)
			continue
		}

		l.Debugln("connect from", conn.RemoteAddr())

		err = dialer.SetTCPOptions(conn.(*net.TCPConn))
		if err != nil {
			l.Infoln(err)
		}

		tc := tls.Server(conn, tlsCfg)
		err = tc.Handshake()
		if err != nil {
			l.Infoln("TLS handshake (BEP/tcp):", err)
			tc.Close()
			continue
		}

		conns <- model.IntermediateConnection{
			tc, model.ConnectionTypeDirectAccept,
		}
	}
}
<commit_msg>lib/connections: Allow "tcp4" and "tcp6" addresses<commit_after>// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package connections

import (
	"crypto/tls"
	"net"
	"net/url"
	"strings"

	"github.com/syncthing/syncthing/lib/dialer"
	"github.com/syncthing/syncthing/lib/model"
)

func init() {
	for _, network := range []string{"tcp", "tcp4", "tcp6"} {
		dialers[network] = makeTcpDialer(network)
		listeners[network] = makeTcpListener(network)
	}

}

func makeTcpDialer(network string) DialerFactory {
	return func(uri *url.URL, tlsCfg *tls.Config) (*tls.Conn, error) {
		// Check that there is a port number in uri.Host, otherwise add one.
		host, port, err := net.SplitHostPort(uri.Host)
		if err != nil && strings.HasPrefix(err.Error(), "missing port") {
			// addr is on the form "1.2.3.4"
			uri.Host = net.JoinHostPort(uri.Host, "22000")
		} else if err == nil && port == "" {
			// addr is on the form "1.2.3.4:"
			uri.Host = net.JoinHostPort(host, "22000")
		}

		// Don't try to resolve the address before dialing. The dialer may be a
		// proxy, and we should let the proxy do the resolving in that case.
		conn, err := dialer.Dial(network, uri.Host)
		if err != nil {
			l.Debugln(err)
			return nil, err
		}

		tc := tls.Client(conn, tlsCfg)
		err = tc.Handshake()
		if err != nil {
			tc.Close()
			return nil, err
		}

		return tc, nil
	}
}

func makeTcpListener(network string) ListenerFactory {
	return func(uri *url.URL, tlsCfg *tls.Config, conns chan<- model.IntermediateConnection) {
		tcaddr, err := net.ResolveTCPAddr(network, uri.Host)
		if err != nil {
			l.Fatalln("listen (BEP/tcp):", err)
			return
		}
		listener, err := net.ListenTCP(network, tcaddr)
		if err != nil {
			l.Fatalln("listen (BEP/tcp):", err)
			return
		}

		for {
			conn, err := listener.Accept()
			if err != nil {
				l.Warnln("Accepting connection (BEP/tcp):", err)
				continue
			}

			l.Debugln("connect from", conn.RemoteAddr())

			err = dialer.SetTCPOptions(conn.(*net.TCPConn))
			if err != nil {
				l.Infoln(err)
			}

			tc := tls.Server(conn, tlsCfg)
			err = tc.Handshake()
			if err != nil {
				l.Infoln("TLS handshake (BEP/tcp):", err)
				tc.Close()
				continue
			}

			conns <- model.IntermediateConnection{
				tc, model.ConnectionTypeDirectAccept,
			}
		}
	}
}
<|endoftext|>


<commit_before>package cni

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/lager"

	"github.com/containernetworking/cni/libcni"
	"github.com/containernetworking/cni/pkg/types"
)

type CNILoader struct {
	PluginDir string
	ConfigDir string
	Logger    lager.Logger
}

func (l *CNILoader) GetCNIConfig() *libcni.CNIConfig {
	return &libcni.CNIConfig{Path: []string{l.PluginDir}}
}

func (l *CNILoader) GetNetworkConfigs() ([]*libcni.NetworkConfig, error) {
	networkConfigs := []*libcni.NetworkConfig{}
	err := filepath.Walk(l.ConfigDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		if !strings.HasSuffix(path, ".conf") {
			return nil
		}

		conf, err := libcni.ConfFromFile(path)
		if err != nil {
			return fmt.Errorf("unable to load config from %s: %s", path, err)
		}

		networkConfigs = append(networkConfigs, conf)

		l.Logger.Info("loaded-config", lager.Data{"network": conf.Network, "raw": string(conf.Bytes)})
		return nil
	})

	if err != nil {
		return networkConfigs, fmt.Errorf("error loading config: %s", err)
	}

	return networkConfigs, nil
}

type CNIController struct {
	Logger lager.Logger

	CNIConfig      *libcni.CNIConfig
	NetworkConfigs []*libcni.NetworkConfig
}

func injectConf(original *libcni.NetworkConfig, key string, newValue interface{}) (*libcni.NetworkConfig, error) {
	config := make(map[string]interface{})
	err := json.Unmarshal(original.Bytes, &config)
	if err != nil {
		return nil, fmt.Errorf("unmarshal existing network bytes: %s", err)
	}

	if key == "" {
		return nil, fmt.Errorf("key value can not be empty")
	}

	if newValue == nil {
		return nil, fmt.Errorf("newValue must be specified")
	}

	config[key] = newValue

	newBytes, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	return libcni.ConfFromBytes(newBytes)
}

func (c *CNIController) Up(namespacePath, handle string, properties map[string]string) (*types.Result, error) {
	var result *types.Result
	var err error

	for i, networkConfig := range c.NetworkConfigs {
		runtimeConfig := &libcni.RuntimeConf{
			ContainerID: handle,
			NetNS:       namespacePath,
			IfName:      fmt.Sprintf("eth%d", i),
		}

		if len(properties) > 0 {
			networkConfig, err = injectConf(networkConfig, "network", map[string]interface{}{
				"properties": properties,
			})
			if err != nil {
				return nil, fmt.Errorf("adding garden properties to CNI config: %s", err)
			}
		}

		result, err = c.CNIConfig.AddNetwork(networkConfig, runtimeConfig)
		if err != nil {
			return nil, fmt.Errorf("add network failed: %s", err)
		}
		c.Logger.Info("up-result", lager.Data{"name": networkConfig.Network.Name, "type": networkConfig.Network.Type, "result": result.String()})
	}
	c.Logger.Info("up-complete", lager.Data{"numConfigs": len(c.NetworkConfigs)})

	return result, nil
}

func (c *CNIController) Down(namespacePath, handle string) error {
	var err error
	for i, networkConfig := range c.NetworkConfigs {
		runtimeConfig := &libcni.RuntimeConf{
			ContainerID: handle,
			NetNS:       namespacePath,
			IfName:      fmt.Sprintf("eth%d", i),
		}

		err = c.CNIConfig.DelNetwork(networkConfig, runtimeConfig)
		if err != nil {
			return fmt.Errorf("del network failed: %s", err)
		}

		c.Logger.Info("down-result", lager.Data{"name": networkConfig.Network.Name, "type": networkConfig.Network.Type})
	}
	c.Logger.Info("down-complete", lager.Data{"numConfigs": len(c.NetworkConfigs)})

	return nil
}
<commit_msg>Use the infectConf from the cni library<commit_after>package cni

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/lager"

	"github.com/containernetworking/cni/libcni"
	"github.com/containernetworking/cni/pkg/types"
)

type CNILoader struct {
	PluginDir string
	ConfigDir string
	Logger    lager.Logger
}

func (l *CNILoader) GetCNIConfig() *libcni.CNIConfig {
	return &libcni.CNIConfig{Path: []string{l.PluginDir}}
}

func (l *CNILoader) GetNetworkConfigs() ([]*libcni.NetworkConfig, error) {
	networkConfigs := []*libcni.NetworkConfig{}
	err := filepath.Walk(l.ConfigDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		if !strings.HasSuffix(path, ".conf") {
			return nil
		}

		conf, err := libcni.ConfFromFile(path)
		if err != nil {
			return fmt.Errorf("unable to load config from %s: %s", path, err)
		}

		networkConfigs = append(networkConfigs, conf)

		l.Logger.Info("loaded-config", lager.Data{"network": conf.Network, "raw": string(conf.Bytes)})
		return nil
	})

	if err != nil {
		return networkConfigs, fmt.Errorf("error loading config: %s", err)
	}

	return networkConfigs, nil
}

type CNIController struct {
	Logger lager.Logger

	CNIConfig      *libcni.CNIConfig
	NetworkConfigs []*libcni.NetworkConfig
}

func (c *CNIController) Up(namespacePath, handle string, properties map[string]string) (*types.Result, error) {
	var result *types.Result
	var err error

	for i, networkConfig := range c.NetworkConfigs {
		runtimeConfig := &libcni.RuntimeConf{
			ContainerID: handle,
			NetNS:       namespacePath,
			IfName:      fmt.Sprintf("eth%d", i),
		}

		if len(properties) > 0 {
			networkConfig, err = libcni.InjectConf(networkConfig, "network", map[string]interface{}{
				"properties": properties,
			})
			if err != nil {
				return nil, fmt.Errorf("adding garden properties to CNI config: %s", err)
			}
		}

		result, err = c.CNIConfig.AddNetwork(networkConfig, runtimeConfig)
		if err != nil {
			return nil, fmt.Errorf("add network failed: %s", err)
		}
		c.Logger.Info("up-result", lager.Data{"name": networkConfig.Network.Name, "type": networkConfig.Network.Type, "result": result.String()})
	}
	c.Logger.Info("up-complete", lager.Data{"numConfigs": len(c.NetworkConfigs)})

	return result, nil
}

func (c *CNIController) Down(namespacePath, handle string) error {
	var err error
	for i, networkConfig := range c.NetworkConfigs {
		runtimeConfig := &libcni.RuntimeConf{
			ContainerID: handle,
			NetNS:       namespacePath,
			IfName:      fmt.Sprintf("eth%d", i),
		}

		err = c.CNIConfig.DelNetwork(networkConfig, runtimeConfig)
		if err != nil {
			return fmt.Errorf("del network failed: %s", err)
		}

		c.Logger.Info("down-result", lager.Data{"name": networkConfig.Network.Name, "type": networkConfig.Network.Type})
	}
	c.Logger.Info("down-complete", lager.Data{"numConfigs": len(c.NetworkConfigs)})

	return nil
}
<|endoftext|>