<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"log"
	"os"
	"path/filepath"
	"regexp"
)

type jsonObjectRaw map[string]json.RawMessage
type jsonObject map[string]interface{}

type tilePosition struct {
	collection string
	path       string
}

// Log to standard error
var stderr = log.New(os.Stderr, "", 0)

var nameSelect = regexp.MustCompile(`[^:/\\]+$`)
var tilepathCut = regexp.MustCompile(`:/{0,2}`)

func getJSONRawSlice(r jsonObjectRaw, k string) (*[]jsonObjectRaw, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var rawSlice []jsonObjectRaw
	if err := json.Unmarshal(blob, &rawSlice); err != nil {
		return nil, fmt.Errorf("JSON decode failed: %w", err)
	}
	return &rawSlice, nil
}

func getJSONSlice(r jsonObjectRaw, k string) (*[]jsonObject, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedSlice []jsonObject
	if err := json.Unmarshal(blob, &decodedSlice); err != nil {
		return nil, fmt.Errorf("JSON decode failed: %w", err)
	}
	return &decodedSlice, nil
}

func getJSONRawObject(r jsonObjectRaw, k string) (*jsonObjectRaw, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedObject jsonObjectRaw

	if err := json.Unmarshal(blob, &decodedObject); err != nil {
		return nil, fmt.Errorf("JSON decode failed: %w", err)
	}
	return &decodedObject, nil
}

func readMapFile(path string) (*jsonObjectRaw, error) {
	// Read the file
	mapBlob, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read failed: %w", err)
	}
	// Decode it in hexMap
	var hexMap jsonObjectRaw
	if err = json.Unmarshal(mapBlob, &hexMap); err != nil {
		return nil, fmt.Errorf("JSON decode failed: %w", err)
	}
	return &hexMap, nil
}

func readSettingsBlob(userConfig string) ([]byte, error) {
	settings := filepath.Join(userConfig, "hex-kit", "Settings")
	stderr.Print("Reading user settings from: ", settings)
	settingsBlob, err := os.ReadFile(settings)
	return settingsBlob, err
}

func getSettings() (jsonObjectRaw, error) {
	var userConfig string
	var settingsBlob []byte
	var settingsRaw jsonObjectRaw
	var err error
	userConfig, err = os.UserConfigDir()
	if err != nil {
		return nil, err
	}
	settingsBlob, err = readSettingsBlob(userConfig)
	if err != nil {
		return nil, fmt.Errorf("unable to read settings: %w", err)
	}
	err = json.Unmarshal(settingsBlob, &settingsRaw)
	if err != nil {
		return nil, fmt.Errorf("unable to decode settings: %w", err)
	}
	return settingsRaw, nil
}

// Search fon all png files under the current path
func pathMap(collectionName, basePath string, fileList *map[string][]tilePosition) fs.WalkDirFunc {
	return func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			stderr.Println("Warning: while searching for PNG files: ", err)
			return nil
		}
		tileName := d.Name()
		lenPath := len(path)
		if lenPath > 4 && path[lenPath-4:] == ".png" {
			relPathTile, _ := filepath.Rel(basePath, path)
			var tp tilePosition
			tp.collection = collectionName
			tp.path = filepath.ToSlash(filepath.Clean(relPathTile))
			(*fileList)[tileName] = append((*fileList)[tileName], tp)
		}
		return nil
	}
}

func getCollectionDir(settings jsonObjectRaw) (*map[string]string, error) {
	// Build the list of collections
	collectionsDir := make(map[string]string)
	collections, err := getJSONRawObject(settings, "tiles")
	if err != nil {
		return nil, fmt.Errorf("unable to access the \"tiles\" list: %w", err)
	}
	for name, collectionBlob := range *collections {
		var collection jsonObject
		if err := json.Unmarshal(collectionBlob, &collection); err != nil {
			return nil, fmt.Errorf("unable to parse tiles[%s]: %w", name, err)
		}
		// Ignore source if hidden
		hiddenIntf, ok := collection["hidden"]
		if ok {
			hidden, isBool := hiddenIntf.(bool)
			if isBool && hidden {
				continue
			}
		}
		pathIntf, ok := collection["path"]
		if !ok {
			return nil, fmt.Errorf("no path for %s: %w", name, err)
		}
		path, ok := pathIntf.(string)
		if !ok {
			return nil, fmt.Errorf("the path for %s is not a string: %w", name, err)
		}
		// Relative collection path
		if !filepath.IsAbs(path) {
			collectionsDir[name] = filepath.Join(os.Args[1], "resources", "app.asar.unpacked", path)
			continue
		}
		// Absolute collection path
		collectionsDir[name] = path
	}
	return &collectionsDir, nil

}

func tileUpdate(t *jsonObject, fileList map[string][]tilePosition) (modified bool, err error) {
	sourceBlob, ok := (*t)["source"]
	if !ok {
		return false, errors.New("no tile source found")
	}
	source, ok := sourceBlob.(string)
	if !ok {
		return false, errors.New("incorrect tile source (not a string)")
	}
	// Skip the default blank tiles
	if source[:6] == "Blank:" {
		return false, nil
	}
	// Have we found the tile
	fileName := nameSelect.FindString(source)
	pathList, ok := fileList[fileName]
	if !ok {
		return false, fmt.Errorf("unknown tile %s (%s)", fileName, source)
	}
	// Search for the current path in the
	firstSplit := tilepathCut.Split(source, 2)
	if len(firstSplit) < 2 {
		return false, fmt.Errorf("no \":\" in source (%s)", source)
	}
	targetCollection := firstSplit[0]
	targetPath := firstSplit[1]
	var bestScore int
	var selected tilePosition
pathSearch:
	for _, p := range pathList {
		// The tiles still exists at the same place. We keep it.
		if targetCollection == p.collection && targetPath == p.path {
			break pathSearch
		}
		currentScore := len(p.path)
		// A tile in the same collection will be preferred
		if targetCollection == p.collection && (currentScore+256) > bestScore {
			bestScore = currentScore + 256
			selected = p
		}
		// Otherwise, take the longer path
		if currentScore > bestScore {
			bestScore = currentScore
			selected = p
		}
	}
	// A new value was found: update the source
	if selected.collection != "" && selected.path != "" {
		modified = true
		(*t)["source"] = selected.collection + ":/" + selected.path
	}
	return modified, nil

}

func updateMapFile(mapFile *jsonObjectRaw, fileList map[string][]tilePosition) error {
	// Get the layers list
	layers, err := getJSONRawSlice(*mapFile, "layers")
	if err != nil {
		return fmt.Errorf("Map format error: %w", err)
	}
	// Search each layer
	layersModified := false
	for i, v := range *layers {
		tiles, err := getJSONSlice(v, "tiles")
		if err != nil {
			return fmt.Errorf("Layer %d: Map format error: %w", i+1, err)
		}
		// Update all tiles
		tilesModified := false
		for j, t := range *tiles {
			// Ignore undefined tiles
			if t == nil {
				continue
			}
			modified, err := tileUpdate(&t, fileList)
			if err != nil {
				stderr.Println("Warning: layer", i+1, "tile", j+1, ":", err)
				continue
			}
			tilesModified = tilesModified || modified
		}
		if tilesModified {
			tilesBlob, err := json.Marshal(tiles)
			if err != nil {
				return err
			}
			v["tiles"] = tilesBlob
			layersModified = true
		}
	}
	if layersModified {
		layersBlob, err := json.Marshal(layers)
		if err != nil {
			return err
		}
		(*mapFile)["layers"] = layersBlob
	}
	return nil
}

func main() {
	if len(os.Args) != 3 {
		stderr.Println("Usage:", os.Args[0], "HexkitPath MapPath")
		return
	}
	// Read the settngs and get the
	settings, err := getSettings()
	if err != nil {
		stderr.Fatal("Unable to read user settings: ", err)
	}
	collectionsDir, err := getCollectionDir(settings)
	if err != nil {
		stderr.Fatal("Unable to read the list of collections: ", err)
	}
	// Build the list of PNG files
	fileList := make(map[string][]tilePosition, 4096)
	for name, path := range *collectionsDir {
		if err := filepath.WalkDir(path, pathMap(name, path, &fileList)); err != nil {
			stderr.Fatal(err)
		}
	}
	// Read the file
	hexMap, err := readMapFile(os.Args[2])
	if err != nil {
		stderr.Fatal("Error: unable to read map file: ", err)
	}
	if err = updateMapFile(hexMap, fileList); err != nil {
		stderr.Fatal(err)
	}
	b, err := json.Marshal(*hexMap)
	if err != nil {
		stderr.Fatal(err)
	}
	_, err = os.Stdout.Write(b)
	if err != nil {
		stderr.Fatal(err)
	}
}
<commit_msg>Formated with golines and gofumpt<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"log"
	"os"
	"path/filepath"
	"regexp"
)

type (
	jsonObjectRaw map[string]json.RawMessage
	jsonObject    map[string]interface{}
)

type tilePosition struct {
	collection string
	path       string
}

// Log to standard error
var stderr = log.New(os.Stderr, "", 0)

var (
	nameSelect  = regexp.MustCompile(`[^:/\\]+$`)
	tilepathCut = regexp.MustCompile(`:/{0,2}`)
)

func getJSONRawSlice(r jsonObjectRaw, k string) (*[]jsonObjectRaw, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var rawSlice []jsonObjectRaw
	if err := json.Unmarshal(blob, &rawSlice); err != nil {
		return nil, fmt.Errorf("JSON decode failed: %w", err)
	}
	return &rawSlice, nil
}

func getJSONSlice(r jsonObjectRaw, k string) (*[]jsonObject, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedSlice []jsonObject
	if err := json.Unmarshal(blob, &decodedSlice); err != nil {
		return nil, fmt.Errorf("JSON decode failed: %w", err)
	}
	return &decodedSlice, nil
}

func getJSONRawObject(r jsonObjectRaw, k string) (*jsonObjectRaw, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedObject jsonObjectRaw

	if err := json.Unmarshal(blob, &decodedObject); err != nil {
		return nil, fmt.Errorf("JSON decode failed: %w", err)
	}
	return &decodedObject, nil
}

func readMapFile(path string) (*jsonObjectRaw, error) {
	// Read the file
	mapBlob, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read failed: %w", err)
	}
	// Decode it in hexMap
	var hexMap jsonObjectRaw
	if err = json.Unmarshal(mapBlob, &hexMap); err != nil {
		return nil, fmt.Errorf("JSON decode failed: %w", err)
	}
	return &hexMap, nil
}

func readSettingsBlob(userConfig string) ([]byte, error) {
	settings := filepath.Join(userConfig, "hex-kit", "Settings")
	stderr.Print("Reading user settings from: ", settings)
	settingsBlob, err := os.ReadFile(settings)
	return settingsBlob, err
}

func getSettings() (jsonObjectRaw, error) {
	var userConfig string
	var settingsBlob []byte
	var settingsRaw jsonObjectRaw
	var err error
	userConfig, err = os.UserConfigDir()
	if err != nil {
		return nil, err
	}
	settingsBlob, err = readSettingsBlob(userConfig)
	if err != nil {
		return nil, fmt.Errorf("unable to read settings: %w", err)
	}
	err = json.Unmarshal(settingsBlob, &settingsRaw)
	if err != nil {
		return nil, fmt.Errorf("unable to decode settings: %w", err)
	}
	return settingsRaw, nil
}

// Search fon all png files under the current path
func pathMap(collectionName, basePath string, fileList *map[string][]tilePosition) fs.WalkDirFunc {
	return func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			stderr.Println("Warning: while searching for PNG files: ", err)
			return nil
		}
		tileName := d.Name()
		lenPath := len(path)
		if lenPath > 4 && path[lenPath-4:] == ".png" {
			relPathTile, _ := filepath.Rel(basePath, path)
			var tp tilePosition
			tp.collection = collectionName
			tp.path = filepath.ToSlash(filepath.Clean(relPathTile))
			(*fileList)[tileName] = append((*fileList)[tileName], tp)
		}
		return nil
	}
}

func getCollectionDir(settings jsonObjectRaw) (*map[string]string, error) {
	// Build the list of collections
	collectionsDir := make(map[string]string)
	collections, err := getJSONRawObject(settings, "tiles")
	if err != nil {
		return nil, fmt.Errorf("unable to access the \"tiles\" list: %w", err)
	}
	for name, collectionBlob := range *collections {
		var collection jsonObject
		if err := json.Unmarshal(collectionBlob, &collection); err != nil {
			return nil, fmt.Errorf("unable to parse tiles[%s]: %w", name, err)
		}
		// Ignore source if hidden
		hiddenIntf, ok := collection["hidden"]
		if ok {
			hidden, isBool := hiddenIntf.(bool)
			if isBool && hidden {
				continue
			}
		}
		pathIntf, ok := collection["path"]
		if !ok {
			return nil, fmt.Errorf("no path for %s: %w", name, err)
		}
		path, ok := pathIntf.(string)
		if !ok {
			return nil, fmt.Errorf("the path for %s is not a string: %w", name, err)
		}
		// Relative collection path
		if !filepath.IsAbs(path) {
			collectionsDir[name] = filepath.Join(os.Args[1], "resources", "app.asar.unpacked", path)
			continue
		}
		// Absolute collection path
		collectionsDir[name] = path
	}
	return &collectionsDir, nil
}

func tileUpdate(t *jsonObject, fileList map[string][]tilePosition) (modified bool, err error) {
	sourceBlob, ok := (*t)["source"]
	if !ok {
		return false, errors.New("no tile source found")
	}
	source, ok := sourceBlob.(string)
	if !ok {
		return false, errors.New("incorrect tile source (not a string)")
	}
	// Skip the default blank tiles
	if source[:6] == "Blank:" {
		return false, nil
	}
	// Have we found the tile
	fileName := nameSelect.FindString(source)
	pathList, ok := fileList[fileName]
	if !ok {
		return false, fmt.Errorf("unknown tile %s (%s)", fileName, source)
	}
	// Search for the current path in the
	firstSplit := tilepathCut.Split(source, 2)
	if len(firstSplit) < 2 {
		return false, fmt.Errorf("no \":\" in source (%s)", source)
	}
	targetCollection := firstSplit[0]
	targetPath := firstSplit[1]
	var bestScore int
	var selected tilePosition
pathSearch:
	for _, p := range pathList {
		// The tiles still exists at the same place. We keep it.
		if targetCollection == p.collection && targetPath == p.path {
			break pathSearch
		}
		currentScore := len(p.path)
		// A tile in the same collection will be preferred
		if targetCollection == p.collection && (currentScore+256) > bestScore {
			bestScore = currentScore + 256
			selected = p
		}
		// Otherwise, take the longer path
		if currentScore > bestScore {
			bestScore = currentScore
			selected = p
		}
	}
	// A new value was found: update the source
	if selected.collection != "" && selected.path != "" {
		modified = true
		(*t)["source"] = selected.collection + ":/" + selected.path
	}
	return modified, nil
}

func updateMapFile(mapFile *jsonObjectRaw, fileList map[string][]tilePosition) error {
	// Get the layers list
	layers, err := getJSONRawSlice(*mapFile, "layers")
	if err != nil {
		return fmt.Errorf("Map format error: %w", err)
	}
	// Search each layer
	layersModified := false
	for i, v := range *layers {
		tiles, err := getJSONSlice(v, "tiles")
		if err != nil {
			return fmt.Errorf("Layer %d: Map format error: %w", i+1, err)
		}
		// Update all tiles
		tilesModified := false
		for j, t := range *tiles {
			// Ignore undefined tiles
			if t == nil {
				continue
			}
			modified, err := tileUpdate(&t, fileList)
			if err != nil {
				stderr.Println("Warning: layer", i+1, "tile", j+1, ":", err)
				continue
			}
			tilesModified = tilesModified || modified
		}
		if tilesModified {
			tilesBlob, err := json.Marshal(tiles)
			if err != nil {
				return err
			}
			v["tiles"] = tilesBlob
			layersModified = true
		}
	}
	if layersModified {
		layersBlob, err := json.Marshal(layers)
		if err != nil {
			return err
		}
		(*mapFile)["layers"] = layersBlob
	}
	return nil
}

func main() {
	if len(os.Args) != 3 {
		stderr.Println("Usage:", os.Args[0], "HexkitPath MapPath")
		return
	}
	// Read the settngs and get the
	settings, err := getSettings()
	if err != nil {
		stderr.Fatal("Unable to read user settings: ", err)
	}
	collectionsDir, err := getCollectionDir(settings)
	if err != nil {
		stderr.Fatal("Unable to read the list of collections: ", err)
	}
	// Build the list of PNG files
	fileList := make(map[string][]tilePosition, 4096)
	for name, path := range *collectionsDir {
		if err := filepath.WalkDir(path, pathMap(name, path, &fileList)); err != nil {
			stderr.Fatal(err)
		}
	}
	// Read the file
	hexMap, err := readMapFile(os.Args[2])
	if err != nil {
		stderr.Fatal("Error: unable to read map file: ", err)
	}
	if err = updateMapFile(hexMap, fileList); err != nil {
		stderr.Fatal(err)
	}
	b, err := json.Marshal(*hexMap)
	if err != nil {
		stderr.Fatal(err)
	}
	_, err = os.Stdout.Write(b)
	if err != nil {
		stderr.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package hhfrag

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/tabwriter"

	"github.com/BurntSushi/bcbgo/apps/hhsuite"
	"github.com/BurntSushi/bcbgo/io/hhm"
	"github.com/BurntSushi/bcbgo/io/hhr"
	"github.com/BurntSushi/bcbgo/io/pdb"
	"github.com/BurntSushi/bcbgo/seq"
)

type PDBDatabase hhsuite.Database

func (db PDBDatabase) HHsuite() hhsuite.Database {
	resolved := hhsuite.Database(db).Resolve()
	dbName := path.Base(resolved)
	return hhsuite.Database(path.Join(resolved, dbName))
}

func (db PDBDatabase) PDB() string {
	resolved := hhsuite.Database(db).Resolve()
	return path.Join(resolved, "pdb")
}

type Fragments struct {
	Frags      []Fragment
	Start, End int
}

// better returns true if f1 is 'better' than f2. Otherwise false.
func (f1 Fragments) better(f2 Fragments) bool {
	return len(f1.Frags) >= len(f2.Frags)
}

func (frags Fragments) Write(w io.Writer) {
	tabw := tabwriter.NewWriter(w, 0, 4, 4, ' ', 0)
	fmt.Fprintln(tabw, "Hit\tQuery\tTemplate\tProb\tCorrupt")
	for _, frag := range frags.Frags {
		var corruptStr string
		if frag.IsCorrupt() {
			corruptStr = "\tcorrupt"
		}
		fmt.Fprintf(tabw, "%s\t(%d-%d)\t(%d-%d)\t%f%s\n",
			frag.Template.Name,
			frag.Hit.QueryStart, frag.Hit.QueryEnd,
			frag.Hit.TemplateStart, frag.Hit.TemplateEnd,
			frag.Hit.Prob,
			corruptStr)
	}
	tabw.Flush()
}

func FindFragments(pdbDb PDBDatabase, blits bool,
	queryHHM *hhm.HHM, qs seq.Sequence, start, end int) (*Fragments, error) {

	pre := fmt.Sprintf("bcbgo-hhfrag-hhm-%d-%d_", start, end)
	hhmFile, err := ioutil.TempFile("", pre)
	if err != nil {
		return nil, err
	}
	defer os.Remove(hhmFile.Name())
	hhmName := hhmFile.Name()

	if err := hhm.Write(hhmFile, queryHHM.Slice(start, end)); err != nil {
		return nil, err
	}

	var results *hhr.HHR
	if blits {
		results, err = hhsuite.HHBlitsDefault.Run(pdbDb.HHsuite(), hhmName)
	} else {
		results, err = hhsuite.HHSearchDefault.Run(pdbDb.HHsuite(), hhmName)
	}
	if err != nil {
		return nil, err
	}

	frags := make([]Fragment, len(results.Hits))
	for i, hit := range results.Hits {
		hit.QueryStart += start
		hit.QueryEnd += start
		frag, err := NewFragment(pdbDb, qs, hit)
		if err != nil {
			return nil, err
		}
		frags[i] = frag
	}
	return &Fragments{
		Frags: frags,
		Start: start,
		End:   end,
	}, nil
}

// An HHfrag Fragment corresponds to a match between a portion of a query
// HMM and a portion of a template HMM. The former is represented as a slice
// of a regular sequence, where the latter is represented as an hhsuite hit
// and a list of alpha-carbon atoms corresponding to the matched region.
type Fragment struct {
	Query    seq.Sequence
	Template seq.Sequence
	Hit      hhr.Hit
	CaAtoms  pdb.Atoms
}

// IsCorrupt returns true when a particular fragment could not be paired
// with alpha-carbon positions for every residue in the template strand.
// (This problem stems from the fact that we use SEQRES records for sequence
// information, but not all residues in SEQRES have alpha-carbon ATOM records
// associated with them.)
func (frag Fragment) IsCorrupt() bool {
	return frag.CaAtoms == nil
}

// NewFragment constructs a new fragment from a full query sequence and the
// hit from the HHR file.
//
// Since NewFragment requires access to the raw PDB alpha-carbon atoms (and
// the sequence) of the template hit, you'll also need to pass a path to the
// PDB database. (Which is a directory containing a flat list of all
// PDB files used to construct the corresponding hhblits database.) This
// database is usually located inside the 'pdb' directory contained in the
// corresponding hhsuite database. i.e., $HHLIB/data/pdb-select25/pdb
func NewFragment(
	pdbDb PDBDatabase, qs seq.Sequence, hit hhr.Hit) (Fragment, error) {

	pdbName := getTemplatePdbName(hit.Name)
	pdbEntry, err := pdb.New(path.Join(
		pdbDb.PDB(), fmt.Sprintf("%s.pdb", pdbName)))
	if err != nil {
		return Fragment{}, err
	}

	// Load in the sequence from the PDB file using the SEQRES residues.
	ts, te := hit.TemplateStart, hit.TemplateEnd
	chain := pdbEntry.OneChain()
	tseq := seq.Sequence{
		Name:     pdbName,
		Residues: make([]seq.Residue, te-ts+1),
	}

	// We copy here to avoid pinning pdb.Entry objects.
	copy(tseq.Residues, chain.Sequence[ts-1:te])

	frag := Fragment{
		Query:    qs.Slice(hit.QueryStart-1, hit.QueryEnd),
		Template: tseq,
		Hit:      hit,
		CaAtoms:  nil,
	}

	// We designate "corrupt" if the query/template hit regions are of
	// different length. i.e., we don't allow gaps (yet).
	// BUG(burntsushi): Fragments with gaps are marked as corrupt.
	if hit.QueryEnd-hit.QueryStart != hit.TemplateEnd-hit.TemplateStart {
		return frag, nil
	}

	// We also designate "corrupt" if there are any gaps in our alpha-carbon
	// atom list.
	atoms := chain.CaAtomSlice(ts-1, te)
	if atoms == nil {
		return frag, nil
	}

	// One again, we copy to avoid pinning memory.
	frag.CaAtoms = make(pdb.Atoms, len(atoms))
	copy(frag.CaAtoms, atoms)

	return frag, nil
}

func getTemplatePdbName(hitName string) string {
	return strings.SplitN(strings.TrimSpace(hitName), " ", 2)[0]
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<commit_msg>Only use 1 CPU in hhfrag, since we're parallelizing at a higher level.<commit_after>package hhfrag

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/tabwriter"

	"github.com/BurntSushi/bcbgo/apps/hhsuite"
	"github.com/BurntSushi/bcbgo/io/hhm"
	"github.com/BurntSushi/bcbgo/io/hhr"
	"github.com/BurntSushi/bcbgo/io/pdb"
	"github.com/BurntSushi/bcbgo/seq"
)

type PDBDatabase hhsuite.Database

func (db PDBDatabase) HHsuite() hhsuite.Database {
	resolved := hhsuite.Database(db).Resolve()
	dbName := path.Base(resolved)
	return hhsuite.Database(path.Join(resolved, dbName))
}

func (db PDBDatabase) PDB() string {
	resolved := hhsuite.Database(db).Resolve()
	return path.Join(resolved, "pdb")
}

type Fragments struct {
	Frags      []Fragment
	Start, End int
}

// better returns true if f1 is 'better' than f2. Otherwise false.
func (f1 Fragments) better(f2 Fragments) bool {
	return len(f1.Frags) >= len(f2.Frags)
}

func (frags Fragments) Write(w io.Writer) {
	tabw := tabwriter.NewWriter(w, 0, 4, 4, ' ', 0)
	fmt.Fprintln(tabw, "Hit\tQuery\tTemplate\tProb\tCorrupt")
	for _, frag := range frags.Frags {
		var corruptStr string
		if frag.IsCorrupt() {
			corruptStr = "\tcorrupt"
		}
		fmt.Fprintf(tabw, "%s\t(%d-%d)\t(%d-%d)\t%f%s\n",
			frag.Template.Name,
			frag.Hit.QueryStart, frag.Hit.QueryEnd,
			frag.Hit.TemplateStart, frag.Hit.TemplateEnd,
			frag.Hit.Prob,
			corruptStr)
	}
	tabw.Flush()
}

func FindFragments(pdbDb PDBDatabase, blits bool,
	queryHHM *hhm.HHM, qs seq.Sequence, start, end int) (*Fragments, error) {

	pre := fmt.Sprintf("bcbgo-hhfrag-hhm-%d-%d_", start, end)
	hhmFile, err := ioutil.TempFile("", pre)
	if err != nil {
		return nil, err
	}
	defer os.Remove(hhmFile.Name())
	hhmName := hhmFile.Name()

	if err := hhm.Write(hhmFile, queryHHM.Slice(start, end)); err != nil {
		return nil, err
	}

	var results *hhr.HHR
	if blits {
		conf := hhsuite.HHBlitsDefault
		conf.CPUs = 1
		results, err = conf.Run(pdbDb.HHsuite(), hhmName)
	} else {
		conf := hhsuite.HHSearchDefault
		conf.CPUs = 1
		results, err = conf.Run(pdbDb.HHsuite(), hhmName)
	}
	if err != nil {
		return nil, err
	}

	frags := make([]Fragment, len(results.Hits))
	for i, hit := range results.Hits {
		hit.QueryStart += start
		hit.QueryEnd += start
		frag, err := NewFragment(pdbDb, qs, hit)
		if err != nil {
			return nil, err
		}
		frags[i] = frag
	}
	return &Fragments{
		Frags: frags,
		Start: start,
		End:   end,
	}, nil
}

// An HHfrag Fragment corresponds to a match between a portion of a query
// HMM and a portion of a template HMM. The former is represented as a slice
// of a regular sequence, where the latter is represented as an hhsuite hit
// and a list of alpha-carbon atoms corresponding to the matched region.
type Fragment struct {
	Query    seq.Sequence
	Template seq.Sequence
	Hit      hhr.Hit
	CaAtoms  pdb.Atoms
}

// IsCorrupt returns true when a particular fragment could not be paired
// with alpha-carbon positions for every residue in the template strand.
// (This problem stems from the fact that we use SEQRES records for sequence
// information, but not all residues in SEQRES have alpha-carbon ATOM records
// associated with them.)
func (frag Fragment) IsCorrupt() bool {
	return frag.CaAtoms == nil
}

// NewFragment constructs a new fragment from a full query sequence and the
// hit from the HHR file.
//
// Since NewFragment requires access to the raw PDB alpha-carbon atoms (and
// the sequence) of the template hit, you'll also need to pass a path to the
// PDB database. (Which is a directory containing a flat list of all
// PDB files used to construct the corresponding hhblits database.) This
// database is usually located inside the 'pdb' directory contained in the
// corresponding hhsuite database. i.e., $HHLIB/data/pdb-select25/pdb
func NewFragment(
	pdbDb PDBDatabase, qs seq.Sequence, hit hhr.Hit) (Fragment, error) {

	pdbName := getTemplatePdbName(hit.Name)
	pdbEntry, err := pdb.New(path.Join(
		pdbDb.PDB(), fmt.Sprintf("%s.pdb", pdbName)))
	if err != nil {
		return Fragment{}, err
	}

	// Load in the sequence from the PDB file using the SEQRES residues.
	ts, te := hit.TemplateStart, hit.TemplateEnd
	chain := pdbEntry.OneChain()
	tseq := seq.Sequence{
		Name:     pdbName,
		Residues: make([]seq.Residue, te-ts+1),
	}

	// We copy here to avoid pinning pdb.Entry objects.
	copy(tseq.Residues, chain.Sequence[ts-1:te])

	frag := Fragment{
		Query:    qs.Slice(hit.QueryStart-1, hit.QueryEnd),
		Template: tseq,
		Hit:      hit,
		CaAtoms:  nil,
	}

	// We designate "corrupt" if the query/template hit regions are of
	// different length. i.e., we don't allow gaps (yet).
	// BUG(burntsushi): Fragments with gaps are marked as corrupt.
	if hit.QueryEnd-hit.QueryStart != hit.TemplateEnd-hit.TemplateStart {
		return frag, nil
	}

	// We also designate "corrupt" if there are any gaps in our alpha-carbon
	// atom list.
	atoms := chain.CaAtomSlice(ts-1, te)
	if atoms == nil {
		return frag, nil
	}

	// One again, we copy to avoid pinning memory.
	frag.CaAtoms = make(pdb.Atoms, len(atoms))
	copy(frag.CaAtoms, atoms)

	return frag, nil
}

func getTemplatePdbName(hitName string) string {
	return strings.SplitN(strings.TrimSpace(hitName), " ", 2)[0]
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package organisation

import (
	"errors"
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	Read(id string) (organisation Organisation, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	db *neoism.Database
}

//NewCypherDriver instantiate driver
func NewCypherDriver(db *neoism.Database) CypherDriver {
	return CypherDriver{db}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	results := []struct {
		ID int
	}{}
	query := &neoism.CypherQuery{
		Statement: "MATCH (x) RETURN ID(x) LIMIT 1",
		Result:    &results,
	}
	err := pcw.db.Cypher(query)
	log.Debugf("CheckConnectivity results:%+v  err: %+v", results, err)
	return err
}

type neoChangeEvent struct {
	StartedAt string
	EndedAt   string
}

type neoReadStruct struct {
	O struct {
		ID        string
		Types     []string
		LEICode   string
		PrefLabel string
		Labels    []string
	}
	Parent struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	Ind struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	Sub []struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	PM []struct {
		M struct {
			ID           string
			Types        []string
			PrefLabel    string
			Title        string
			ChangeEvents []neoChangeEvent
		}
		P struct {
			ID        string
			Types     []string
			PrefLabel string
			Labels    []string
		}
	}
}

func (pcw CypherDriver) Read(uuid string) (organisation Organisation, found bool, err error) {
	organisation = Organisation{}
	results := []struct {
		Rs []neoReadStruct
	}{}
	query := &neoism.CypherQuery{
		Statement: `
				MATCH (o:Organisation{uuid:{uuid}})
				OPTIONAL MATCH (o)<-[:HAS_ORGANISATION]-(m:Membership)
				OPTIONAL MATCH (m)-[:HAS_MEMBER]->(p:Person)
				OPTIONAL MATCH (p)<-[rel:MENTIONS]-(poc:Content)-[mo:MENTIONS]->(o)
				WITH    o,
				{ id:p.uuid, types:labels(p), prefLabel:p.prefLabel} as p,
				{ id:m.uuid, prefLabel:m.prefLabel, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}], annCount:COUNT(poc) } as m ORDER BY m.annCount DESC LIMIT 20
				WITH o, collect({m:m, p:p}) as pm
				OPTIONAL MATCH (o)-[:SUB_ORGANISATION_OF]->(parent:Organisation)
				OPTIONAL MATCH (o)<-[:SUB_ORGANISATION_OF]-(sub:Organisation)
				OPTIONAL MATCH (soc:Content)-[mo:MENTIONS]->(sub)
				WITH o, pm,
				{ id:parent.uuid, types:labels(parent), prefLabel:parent.prefLabel} as parent,
				{ id:sub.uuid, types:labels(sub), prefLabel:sub.prefLabel, annCount:COUNT(soc) } as sub ORDER BY sub.annCount DESC LIMIT 10
				WITH o, pm, parent, collect(sub) as subs
				OPTIONAL MATCH (o)-[:HAS_CLASSIFICATION]->(ind:IndustryClassification)
				WITH o, pm, parent, subs,
				{ id:ind.uuid, types:labels(ind), prefLabel:ind.prefLabel} as ind
				WITH pm, parent, subs, ind, { id:o.uuid, types:labels(o), prefLabel:o.prefLabel, labels:o.aliases, leicode:o.leiCode} as o
				return collect ({o:o, ind:ind, parent:parent, subs:subs, pm:pm}) as rs
							`,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}
	err = pcw.db.Cypher(query)
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v\n", uuid, query.Statement, err)
		return Organisation{}, false, fmt.Errorf("Error accessing Organisation datastore for uuid: %s", uuid)
	}
	log.Debugf("CypherResult ReadOrganisation for uuid: %s was: %+v", uuid, results)
	if (len(results)) == 0 || len(results[0].Rs) == 0 {
		return Organisation{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		errMsg := fmt.Sprintf("Multiple organisations found with the same uuid:%s !", uuid)
		log.Error(errMsg)
		return Organisation{}, true, errors.New(errMsg)
	}
	organisation = neoReadStructToOrganisation(results[0].Rs[0])
	log.Debugf("Returning %v", organisation)
	return organisation, true, nil
}

func neoReadStructToOrganisation(neo neoReadStruct) Organisation {
	//TODO find out why we only get two memberships here compared to 17 off PROD graphDB... also, performance of e.g. Barclays
	public := Organisation{}
	public.Thing = &Thing{}
	public.ID = mapper.IDURL(neo.O.ID)
	public.APIURL = mapper.APIURL(neo.O.ID, neo.O.Types)
	public.Types = mapper.TypeURIs(neo.O.Types)
	public.LEICode = neo.O.LEICode
	public.PrefLabel = neo.O.PrefLabel
	if len(neo.O.Labels) > 0 {
		public.Labels = &neo.O.Labels
	}

	if neo.Ind.ID != "" {
		public.IndustryClassification = &IndustryClassification{}
		public.IndustryClassification.Thing = &Thing{}
		public.IndustryClassification.ID = mapper.IDURL(neo.Ind.ID)
		public.IndustryClassification.APIURL = mapper.APIURL(neo.Ind.ID, neo.Ind.Types)
		public.IndustryClassification.PrefLabel = neo.Ind.PrefLabel
	}
	log.Infof("IndustryClassification=%v", public.IndustryClassification)

	if neo.Parent.ID != "" {
		public.Parent = &Parent{}
		public.Parent.Thing = &Thing{}
		public.Parent.ID = mapper.IDURL(neo.Parent.ID)
		public.Parent.APIURL = mapper.APIURL(neo.Parent.ID, neo.Parent.Types)
		public.Parent.Types = mapper.TypeURIs(neo.Parent.Types)
		public.Parent.PrefLabel = neo.Parent.PrefLabel
	}

	if len(neo.Sub) == 1 && neo.Sub[0].ID == "" {
		public.Subsidiaries = make([]Subsidiary, 0, 0)
	} else {
		public.Subsidiaries = make([]Subsidiary, len(neo.Sub))
		for idx, neoSub := range neo.Sub {
			subsidiary := Subsidiary{}
			subsidiary.Thing = &Thing{}
			subsidiary.ID = mapper.IDURL(neoSub.ID)
			subsidiary.APIURL = mapper.APIURL(neoSub.ID, neoSub.Types)
			subsidiary.Types = mapper.TypeURIs(neoSub.Types)
			subsidiary.PrefLabel = neoSub.PrefLabel
			public.Subsidiaries[idx] = subsidiary
		}
	}

	log.Info("LENGTH of memberships:", len(neo.PM))
	if len(neo.PM) == 1 && (neo.PM[0].M.ID == "") {
		public.Memberships = make([]Membership, 0, 0)
	} else {
		public.Memberships = make([]Membership, len(neo.PM))
		for mIdx, neoMem := range neo.PM {
			membership := Membership{}
			membership.Title = neoMem.M.PrefLabel
			membership.Person = Person{}
			membership.Person.Thing = &Thing{}
			membership.Person.ID = mapper.IDURL(neoMem.P.ID)
			membership.Person.APIURL = mapper.APIURL(neoMem.P.ID, neoMem.P.Types)
			membership.Person.Types = mapper.TypeURIs(neoMem.P.Types)
			membership.Person.PrefLabel = neoMem.P.PrefLabel
			if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
				membership.ChangeEvents = b
			}
			public.Memberships[mIdx] = membership
		}
	}
	log.Debugf("neoReadStructToOrganisation neo: %+v result: %+v", neo, public)
	return public
}

func changeEvent(neoChgEvts []neoChangeEvent) (bool, *[]ChangeEvent) {
	var results []ChangeEvent
	if neoChgEvts[0].StartedAt == "" && neoChgEvts[0].EndedAt == "" {
		results = make([]ChangeEvent, 0, 0)
		return false, &results
	}
	for _, neoChgEvt := range neoChgEvts {
		if neoChgEvt.StartedAt != "" {
			results = append(results, ChangeEvent{StartedAt: neoChgEvt.StartedAt})
		}
		if neoChgEvt.EndedAt != "" {
			results = append(results, ChangeEvent{EndedAt: neoChgEvt.EndedAt})
		}
	}
	log.Debugf("changeEvent converted: %+v result:%+v", neoChgEvts, results)
	return true, &results
}
<commit_msg>Sub became Subs in the query - change code to match<commit_after>package organisation

import (
	"errors"
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	Read(id string) (organisation Organisation, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	db *neoism.Database
}

//NewCypherDriver instantiate driver
func NewCypherDriver(db *neoism.Database) CypherDriver {
	return CypherDriver{db}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	results := []struct {
		ID int
	}{}
	query := &neoism.CypherQuery{
		Statement: "MATCH (x) RETURN ID(x) LIMIT 1",
		Result:    &results,
	}
	err := pcw.db.Cypher(query)
	log.Debugf("CheckConnectivity results:%+v  err: %+v", results, err)
	return err
}

type neoChangeEvent struct {
	StartedAt string
	EndedAt   string
}

type neoReadStruct struct {
	O struct {
		ID        string
		Types     []string
		LEICode   string
		PrefLabel string
		Labels    []string
	}
	Parent struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	Ind struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	Subs []struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	PM []struct {
		M struct {
			ID           string
			Types        []string
			PrefLabel    string
			Title        string
			ChangeEvents []neoChangeEvent
		}
		P struct {
			ID        string
			Types     []string
			PrefLabel string
			Labels    []string
		}
	}
}

func (pcw CypherDriver) Read(uuid string) (organisation Organisation, found bool, err error) {
	organisation = Organisation{}
	results := []struct {
		Rs []neoReadStruct
	}{}
	query := &neoism.CypherQuery{
		Statement: `
				MATCH (o:Organisation{uuid:{uuid}})
				OPTIONAL MATCH (o)<-[:HAS_ORGANISATION]-(m:Membership)
				OPTIONAL MATCH (m)-[:HAS_MEMBER]->(p:Person)
				OPTIONAL MATCH (p)<-[rel:MENTIONS]-(poc:Content)-[mo:MENTIONS]->(o)
				WITH    o,
				{ id:p.uuid, types:labels(p), prefLabel:p.prefLabel} as p,
				{ id:m.uuid, prefLabel:m.prefLabel, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}], annCount:COUNT(poc) } as m ORDER BY m.annCount DESC LIMIT 20
				WITH o, collect({m:m, p:p}) as pm
				OPTIONAL MATCH (o)-[:SUB_ORGANISATION_OF]->(parent:Organisation)
				OPTIONAL MATCH (o)<-[:SUB_ORGANISATION_OF]-(sub:Organisation)
				OPTIONAL MATCH (soc:Content)-[mo:MENTIONS]->(sub)
				WITH o, pm,
				{ id:parent.uuid, types:labels(parent), prefLabel:parent.prefLabel} as parent,
				{ id:sub.uuid, types:labels(sub), prefLabel:sub.prefLabel, annCount:COUNT(soc) } as sub ORDER BY sub.annCount DESC LIMIT 10
				WITH o, pm, parent, collect(sub) as subs
				OPTIONAL MATCH (o)-[:HAS_CLASSIFICATION]->(ind:IndustryClassification)
				WITH o, pm, parent, subs,
				{ id:ind.uuid, types:labels(ind), prefLabel:ind.prefLabel} as ind
				WITH pm, parent, subs, ind, { id:o.uuid, types:labels(o), prefLabel:o.prefLabel, labels:o.aliases, leicode:o.leiCode} as o
				return collect ({o:o, ind:ind, parent:parent, subs:subs, pm:pm}) as rs
							`,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}
	err = pcw.db.Cypher(query)
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v\n", uuid, query.Statement, err)
		return Organisation{}, false, fmt.Errorf("Error accessing Organisation datastore for uuid: %s", uuid)
	}
	log.Debugf("CypherResult ReadOrganisation for uuid: %s was: %+v", uuid, results)
	if (len(results)) == 0 || len(results[0].Rs) == 0 {
		return Organisation{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		errMsg := fmt.Sprintf("Multiple organisations found with the same uuid:%s !", uuid)
		log.Error(errMsg)
		return Organisation{}, true, errors.New(errMsg)
	}
	organisation = neoReadStructToOrganisation(results[0].Rs[0])
	log.Debugf("Returning %v", organisation)
	return organisation, true, nil
}

func neoReadStructToOrganisation(neo neoReadStruct) Organisation {
	//TODO find out why we only get two memberships here compared to 17 off PROD graphDB... also, performance of e.g. Barclays
	public := Organisation{}
	public.Thing = &Thing{}
	public.ID = mapper.IDURL(neo.O.ID)
	public.APIURL = mapper.APIURL(neo.O.ID, neo.O.Types)
	public.Types = mapper.TypeURIs(neo.O.Types)
	public.LEICode = neo.O.LEICode
	public.PrefLabel = neo.O.PrefLabel
	if len(neo.O.Labels) > 0 {
		public.Labels = &neo.O.Labels
	}

	if neo.Ind.ID != "" {
		public.IndustryClassification = &IndustryClassification{}
		public.IndustryClassification.Thing = &Thing{}
		public.IndustryClassification.ID = mapper.IDURL(neo.Ind.ID)
		public.IndustryClassification.APIURL = mapper.APIURL(neo.Ind.ID, neo.Ind.Types)
		public.IndustryClassification.PrefLabel = neo.Ind.PrefLabel
	}
	log.Infof("IndustryClassification=%v", public.IndustryClassification)

	if neo.Parent.ID != "" {
		public.Parent = &Parent{}
		public.Parent.Thing = &Thing{}
		public.Parent.ID = mapper.IDURL(neo.Parent.ID)
		public.Parent.APIURL = mapper.APIURL(neo.Parent.ID, neo.Parent.Types)
		public.Parent.Types = mapper.TypeURIs(neo.Parent.Types)
		public.Parent.PrefLabel = neo.Parent.PrefLabel
	}

	if len(neo.Subs) == 1 && neo.Subs[0].ID == "" {
		public.Subsidiaries = make([]Subsidiary, 0, 0)
	} else {
		public.Subsidiaries = make([]Subsidiary, len(neo.Subs))
		for idx, neoSub := range neo.Subs {
			subsidiary := Subsidiary{}
			subsidiary.Thing = &Thing{}
			subsidiary.ID = mapper.IDURL(neoSub.ID)
			subsidiary.APIURL = mapper.APIURL(neoSub.ID, neoSub.Types)
			subsidiary.Types = mapper.TypeURIs(neoSub.Types)
			subsidiary.PrefLabel = neoSub.PrefLabel
			public.Subsidiaries[idx] = subsidiary
		}
	}

	log.Info("LENGTH of memberships:", len(neo.PM))
	if len(neo.PM) == 1 && (neo.PM[0].M.ID == "") {
		public.Memberships = make([]Membership, 0, 0)
	} else {
		public.Memberships = make([]Membership, len(neo.PM))
		for mIdx, neoMem := range neo.PM {
			membership := Membership{}
			membership.Title = neoMem.M.PrefLabel
			membership.Person = Person{}
			membership.Person.Thing = &Thing{}
			membership.Person.ID = mapper.IDURL(neoMem.P.ID)
			membership.Person.APIURL = mapper.APIURL(neoMem.P.ID, neoMem.P.Types)
			membership.Person.Types = mapper.TypeURIs(neoMem.P.Types)
			membership.Person.PrefLabel = neoMem.P.PrefLabel
			if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
				membership.ChangeEvents = b
			}
			public.Memberships[mIdx] = membership
		}
	}
	log.Debugf("neoReadStructToOrganisation neo: %+v result: %+v", neo, public)
	return public
}

func changeEvent(neoChgEvts []neoChangeEvent) (bool, *[]ChangeEvent) {
	var results []ChangeEvent
	if neoChgEvts[0].StartedAt == "" && neoChgEvts[0].EndedAt == "" {
		results = make([]ChangeEvent, 0, 0)
		return false, &results
	}
	for _, neoChgEvt := range neoChgEvts {
		if neoChgEvt.StartedAt != "" {
			results = append(results, ChangeEvent{StartedAt: neoChgEvt.StartedAt})
		}
		if neoChgEvt.EndedAt != "" {
			results = append(results, ChangeEvent{EndedAt: neoChgEvt.EndedAt})
		}
	}
	log.Debugf("changeEvent converted: %+v result:%+v", neoChgEvts, results)
	return true, &results
}
<|endoftext|>


<commit_before>package myslave

import (
	"fmt"

	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/go-metrics"
)

type slaveMetrics struct {
	Lag metrics.Gauge

	TPS    metrics.Meter
	Events metrics.Meter
}

func newMetrics(host string, port uint16) *slaveMetrics {
	m := &slaveMetrics{}

	tag := telemetry.Tag(host, fmt.Sprintf("%d", port), "v1")
	m.Lag = metrics.NewRegisteredGauge(tag+"mysql.binlog.lag", metrics.DefaultRegistry)
	m.TPS = metrics.NewRegisteredMeter(tag+"mysql.binlog.tps", metrics.DefaultRegistry)
	m.Events = metrics.NewRegisteredMeter(tag+"mysql.binlog.evt", metrics.DefaultRegistry)
	return m
}
<commit_msg>normalize the tag<commit_after>package myslave

import (
	"fmt"
	"strings"

	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/go-metrics"
)

type slaveMetrics struct {
	Lag metrics.Gauge

	TPS    metrics.Meter
	Events metrics.Meter
}

func newMetrics(host string, port uint16) *slaveMetrics {
	m := &slaveMetrics{}

	tag := telemetry.Tag(strings.Replace(host, ".", "_", -1), fmt.Sprintf("%d", port), "v1")
	m.Lag = metrics.NewRegisteredGauge(tag+"mysql.binlog.lag", metrics.DefaultRegistry)
	m.TPS = metrics.NewRegisteredMeter(tag+"mysql.binlog.tps", metrics.DefaultRegistry)
	m.Events = metrics.NewRegisteredMeter(tag+"mysql.binlog.evt", metrics.DefaultRegistry)
	return m
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"bytes"
	"code.google.com/p/go.net/html"
	"io"
	"net/url"
	"regexp"
	"strings"
)

func sanitizeLink(u *url.URL, v string) string {
	p, err := u.Parse(v)
	if err != nil {
		return ""
	}
	if !acceptableUriSchemes[p.Scheme] {
		return ""
	}

	return p.String()
}

func sanitizeStyle(v string) string {
	return v
}

func sanitizeAttributes(u *url.URL, t *html.Token) {
	var attrs []html.Attribute
	var isLink = false
	for _, a := range t.Attr {
		if a.Key == "target" {
		} else if a.Key == "style" {
			a.Val = sanitizeStyle(a.Val)
			attrs = append(attrs, a)
		} else if acceptableAttributes[a.Key] {
			if a.Key == "href" || a.Key == "src" {
				a.Val = sanitizeLink(u, a.Val)
			}
			if a.Key == "href" {
				isLink = true
			}
			attrs = append(attrs, a)
		}
	}
	if isLink {
		attrs = append(attrs, html.Attribute{
			Key: "target",
			Val: "_blank",
		})
	}
	t.Attr = attrs
}

func Sanitize(s string, u *url.URL) (string, string) {
	r := bytes.NewReader([]byte(strings.TrimSpace(s)))
	z := html.NewTokenizer(r)
	buf := &bytes.Buffer{}
	snip := &bytes.Buffer{}
	skip := 0
	u.RawQuery = ""
	u.Fragment = ""
	for {
		if z.Next() == html.ErrorToken {
			if err := z.Err(); err == io.EOF {
				break
			} else {
				return s, snipper(s)
			}
		}

		t := z.Token()
		if t.Type == html.StartTagToken || t.Type == html.SelfClosingTagToken {
			if !acceptableElements[t.Data] {
				if unacceptableElementsWithEndTag[t.Data] && t.Type != html.SelfClosingTagToken {
					skip += 1
				}
			} else {
				sanitizeAttributes(u, &t)
				buf.WriteString(t.String())
			}
		} else if t.Type == html.EndTagToken {
			if !acceptableElements[t.Data] {
				if unacceptableElementsWithEndTag[t.Data] {
					skip -= 1
				}
			} else {
				buf.WriteString(t.String())
			}
		} else if skip == 0 {
			buf.WriteString(t.String())
			if t.Type == html.TextToken {
				snip.WriteString(t.String())
			}
		}
	}

	return buf.String(), snipper(snip.String())
}

const snipLen = 100

var snipRe = regexp.MustCompile("[\\s]+")

func snipper(s string) string {
	s = snipRe.ReplaceAllString(strings.TrimSpace(s), " ")
	s = html.UnescapeString(s)
	if len(s) <= snipLen {
		return s
	}
	s = s[:snipLen]
	i := strings.LastIndexAny(s, " .-!?")
	if i != -1 {
		return s[:i]
	}
	return cleanNonUTF8(s)
}

// Based on list from MDN's HTML5 element list
// https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/HTML5/HTML5_element_list
var acceptableElements = map[string]bool{
	// Root element
	// "html": true,

	// Document metadata
	// "head":  true,
	// "title": true,
	// "base":  true,
	// "link":  true,
	// "meta":  true,
	// "style": true,

	// Scripting
	"noscript": true,
	// "script":   true,

	// Sections
	// "body":    true,
	"section": true,
	"nav":     true,
	"article": true,
	"aside":   true,
	"h1":      true,
	"h2":      true,
	"h3":      true,
	"h4":      true,
	"h5":      true,
	"h6":      true,
	"header":  true,
	"footer":  true,
	"address": true,
	"main":    true,

	// Grouping content
	"p":          true,
	"hr":         true,
	"pre":        true,
	"blockquote": true,
	"ol":         true,
	"ul":         true,
	"li":         true,
	"dl":         true,
	"dt":         true,
	"dd":         true,
	"figure":     true,
	"figcaption": true,
	"div":        true,

	// Text-level semantics
	"a":      true,
	"em":     true,
	"strong": true,
	"small":  true,
	"s":      true,
	"cite":   true,
	"q":      true,
	"dfn":    true,
	"abbr":   true,
	"data":   true,
	"time":   true,
	"code":   true,
	"var":    true,
	"samp":   true,
	"kbd":    true,
	"sub":    true,
	"sup":    true,
	"i":      true,
	"b":      true,
	"u":      true,
	"mark":   true,
	"ruby":   true,
	"rt":     true,
	"rp":     true,
	"bdi":    true,
	"bdo":    true,
	"span":   true,
	"br":     true,
	"wbr":    true,

	// Edits
	"ins": true,
	"del": true,

	// Embedded content
	"img":    true,
	"iframe": true,
	"embed":  true,
	"object": true,
	"param":  true,
	"video":  true,
	"audio":  true,
	"source": true,
	"track":  true,
	"canvas": true,
	"map":    true,
	"area":   true,
	"svg":    true,
	"math":   true,

	// Tabular data
	"table":    true,
	"caption":  true,
	"colgroup": true,
	"col":      true,
	"tbody":    true,
	"thead":    true,
	"tfoot":    true,
	"tr":       true,
	"td":       true,
	"th":       true,

	// Forms
	"form":     true,
	"fieldset": true,
	"legend":   true,
	"label":    true,
	"input":    true,
	"button":   true,
	"select":   true,
	"datalist": true,
	"optgroup": true,
	"option":   true,
	"textarea": true,
	"keygen":   true,
	"output":   true,
	"progress": true,
	"meter":    true,

	// Interactive elements
	// "details":  true,
	// "summary":  true,
	// "menuitem": true,
	// "menu":     true,
}

var unacceptableElementsWithEndTag = map[string]bool{
	"script": true,
	"applet": true,
	"style":  true,
}

// Based on list from MDN's HTML attribute reference
// https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes
var acceptableAttributes = map[string]bool{
	"accept":         true,
	"accept-charset": true,
	// "accesskey":       true,
	"action":       true,
	"align":        true,
	"alt":          true,
	"async":        true,
	"autocomplete": true,
	// "autofocus":       true,
	// "autoplay":        true,
	"bgcolor":         true,
	"border":          true,
	"buffered":        true,
	"challenge":       true,
	"charset":         true,
	"checked":         true,
	"cite":            true,
	"class":           true,
	"code":            true,
	"codebase":        true,
	"color":           true,
	"cols":            true,
	"colspan":         true,
	"content":         true,
	"contenteditable": true,
	"contextmenu":     true,
	"controls":        true,
	"coords":          true,
	"data":            true,
	"data-custom":     true,
	"datetime":        true,
	"default":         true,
	"defer":           true,
	"dir":             true,
	"dirname":         true,
	"disabled":        true,
	"download":        true,
	"draggable":       true,
	"dropzone":        true,
	"enctype":         true,
	"for":             true,
	"form":            true,
	"headers":         true,
	"height":          true,
	"hidden":          true,
	"high":            true,
	"href":            true,
	"hreflang":        true,
	"http-equiv":      true,
	"icon":            true,
	"id":              true,
	"ismap":           true,
	"itemprop":        true,
	"keytype":         true,
	"kind":            true,
	"label":           true,
	"lang":            true,
	"language":        true,
	"list":            true,
	"loop":            true,
	"low":             true,
	"manifest":        true,
	"max":             true,
	"maxlength":       true,
	"media":           true,
	"method":          true,
	"min":             true,
	"multiple":        true,
	"name":            true,
	"novalidate":      true,
	"open":            true,
	"optimum":         true,
	"pattern":         true,
	"ping":            true,
	"placeholder":     true,
	"poster":          true,
	// "preload":         true,
	"pubdate":    true,
	"radiogroup": true,
	"readonly":   true,
	"rel":        true,
	"required":   true,
	"reversed":   true,
	"rows":       true,
	"rowspan":    true,
	"sandbox":    true,
	"spellcheck": true,
	"scope":      true,
	// "scoped":          true,
	// "seamless":        true,
	"selected": true,
	"shape":    true,
	"size":     true,
	"sizes":    true,
	"span":     true,
	"src":      true,
	// "srcdoc":          true,
	"srclang": true,
	"start":   true,
	// "step":            true,
	"style":   true,
	"summary": true,
	// "tabindex":        true,
	// "target":          true,
	"title": true,
	"type":  true,
	// "usemap":          true,
	"value": true,
	"width": true,
	// "wrap":            true,
}

// Based on list from Wikipedia's URI scheme
// http://en.wikipedia.org/wiki/URI_scheme
var acceptableUriSchemes = map[string]bool{
	"aim":      true,
	"apt":      true,
	"bitcoin":  true,
	"callto":   true,
	"cvs":      true,
	"facetime": true,
	"feed":     true,
	"ftp":      true,
	"git":      true,
	"gopher":   true,
	"gtalk":    true,
	"http":     true,
	"https":    true,
	"imap":     true,
	"irc":      true,
	"itms":     true,
	"jabber":   true,
	"magnet":   true,
	"mailto":   true,
	"mms":      true,
	"msnim":    true,
	"news":     true,
	"nntp":     true,
	"rtmp":     true,
	"rtsp":     true,
	"sftp":     true,
	"skype":    true,
	"svn":      true,
	"ymsgr":    true,
}
<commit_msg>Allow some non standard attrs<commit_after>/*
 * Copyright (c) 2013 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"bytes"
	"code.google.com/p/go.net/html"
	"io"
	"net/url"
	"regexp"
	"strings"
)

func sanitizeLink(u *url.URL, v string) string {
	p, err := u.Parse(v)
	if err != nil {
		return ""
	}
	if !acceptableUriSchemes[p.Scheme] {
		return ""
	}

	return p.String()
}

func sanitizeStyle(v string) string {
	return v
}

func sanitizeAttributes(u *url.URL, t *html.Token) {
	var attrs []html.Attribute
	var isLink = false
	for _, a := range t.Attr {
		if a.Key == "target" {
		} else if a.Key == "style" {
			a.Val = sanitizeStyle(a.Val)
			attrs = append(attrs, a)
		} else if acceptableAttributes[a.Key] {
			if a.Key == "href" || a.Key == "src" {
				a.Val = sanitizeLink(u, a.Val)
			}
			if a.Key == "href" {
				isLink = true
			}
			attrs = append(attrs, a)
		}
	}
	if isLink {
		attrs = append(attrs, html.Attribute{
			Key: "target",
			Val: "_blank",
		})
	}
	t.Attr = attrs
}

func Sanitize(s string, u *url.URL) (string, string) {
	r := bytes.NewReader([]byte(strings.TrimSpace(s)))
	z := html.NewTokenizer(r)
	buf := &bytes.Buffer{}
	snip := &bytes.Buffer{}
	skip := 0
	u.RawQuery = ""
	u.Fragment = ""
	for {
		if z.Next() == html.ErrorToken {
			if err := z.Err(); err == io.EOF {
				break
			} else {
				return s, snipper(s)
			}
		}

		t := z.Token()
		if t.Type == html.StartTagToken || t.Type == html.SelfClosingTagToken {
			if !acceptableElements[t.Data] {
				if unacceptableElementsWithEndTag[t.Data] && t.Type != html.SelfClosingTagToken {
					skip += 1
				}
			} else {
				sanitizeAttributes(u, &t)
				buf.WriteString(t.String())
			}
		} else if t.Type == html.EndTagToken {
			if !acceptableElements[t.Data] {
				if unacceptableElementsWithEndTag[t.Data] {
					skip -= 1
				}
			} else {
				buf.WriteString(t.String())
			}
		} else if skip == 0 {
			buf.WriteString(t.String())
			if t.Type == html.TextToken {
				snip.WriteString(t.String())
			}
		}
	}

	return buf.String(), snipper(snip.String())
}

const snipLen = 100

var snipRe = regexp.MustCompile("[\\s]+")

func snipper(s string) string {
	s = snipRe.ReplaceAllString(strings.TrimSpace(s), " ")
	s = html.UnescapeString(s)
	if len(s) <= snipLen {
		return s
	}
	s = s[:snipLen]
	i := strings.LastIndexAny(s, " .-!?")
	if i != -1 {
		return s[:i]
	}
	return cleanNonUTF8(s)
}

// Based on list from MDN's HTML5 element list
// https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/HTML5/HTML5_element_list
var acceptableElements = map[string]bool{
	// Root element
	// "html": true,

	// Document metadata
	// "head":  true,
	// "title": true,
	// "base":  true,
	// "link":  true,
	// "meta":  true,
	// "style": true,

	// Scripting
	"noscript": true,
	// "script":   true,

	// Sections
	// "body":    true,
	"section": true,
	"nav":     true,
	"article": true,
	"aside":   true,
	"h1":      true,
	"h2":      true,
	"h3":      true,
	"h4":      true,
	"h5":      true,
	"h6":      true,
	"header":  true,
	"footer":  true,
	"address": true,
	"main":    true,

	// Grouping content
	"p":          true,
	"hr":         true,
	"pre":        true,
	"blockquote": true,
	"ol":         true,
	"ul":         true,
	"li":         true,
	"dl":         true,
	"dt":         true,
	"dd":         true,
	"figure":     true,
	"figcaption": true,
	"div":        true,

	// Text-level semantics
	"a":      true,
	"em":     true,
	"strong": true,
	"small":  true,
	"s":      true,
	"cite":   true,
	"q":      true,
	"dfn":    true,
	"abbr":   true,
	"data":   true,
	"time":   true,
	"code":   true,
	"var":    true,
	"samp":   true,
	"kbd":    true,
	"sub":    true,
	"sup":    true,
	"i":      true,
	"b":      true,
	"u":      true,
	"mark":   true,
	"ruby":   true,
	"rt":     true,
	"rp":     true,
	"bdi":    true,
	"bdo":    true,
	"span":   true,
	"br":     true,
	"wbr":    true,

	// Edits
	"ins": true,
	"del": true,

	// Embedded content
	"img":    true,
	"iframe": true,
	"embed":  true,
	"object": true,
	"param":  true,
	"video":  true,
	"audio":  true,
	"source": true,
	"track":  true,
	"canvas": true,
	"map":    true,
	"area":   true,
	"svg":    true,
	"math":   true,

	// Tabular data
	"table":    true,
	"caption":  true,
	"colgroup": true,
	"col":      true,
	"tbody":    true,
	"thead":    true,
	"tfoot":    true,
	"tr":       true,
	"td":       true,
	"th":       true,

	// Forms
	"form":     true,
	"fieldset": true,
	"legend":   true,
	"label":    true,
	"input":    true,
	"button":   true,
	"select":   true,
	"datalist": true,
	"optgroup": true,
	"option":   true,
	"textarea": true,
	"keygen":   true,
	"output":   true,
	"progress": true,
	"meter":    true,

	// Interactive elements
	// "details":  true,
	// "summary":  true,
	// "menuitem": true,
	// "menu":     true,
}

var unacceptableElementsWithEndTag = map[string]bool{
	"script": true,
	"applet": true,
	"style":  true,
}

// Based on list from MDN's HTML attribute reference
// https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes
var acceptableAttributes = map[string]bool{
	"accept":         true,
	"accept-charset": true,
	// "accesskey":       true,
	"action":       true,
	"align":        true,
	"alt":          true,
	"async":        true,
	"autocomplete": true,
	// "autofocus":       true,
	// "autoplay":        true,
	"bgcolor":         true,
	"border":          true,
	"buffered":        true,
	"challenge":       true,
	"charset":         true,
	"checked":         true,
	"cite":            true,
	"class":           true,
	"code":            true,
	"codebase":        true,
	"color":           true,
	"cols":            true,
	"colspan":         true,
	"content":         true,
	"contenteditable": true,
	"contextmenu":     true,
	"controls":        true,
	"coords":          true,
	"data":            true,
	"data-custom":     true,
	"datetime":        true,
	"default":         true,
	"defer":           true,
	"dir":             true,
	"dirname":         true,
	"disabled":        true,
	"download":        true,
	"draggable":       true,
	"dropzone":        true,
	"enctype":         true,
	"for":             true,
	"form":            true,
	"headers":         true,
	"height":          true,
	"hidden":          true,
	"high":            true,
	"href":            true,
	"hreflang":        true,
	"http-equiv":      true,
	"icon":            true,
	"id":              true,
	"ismap":           true,
	"itemprop":        true,
	"keytype":         true,
	"kind":            true,
	"label":           true,
	"lang":            true,
	"language":        true,
	"list":            true,
	"loop":            true,
	"low":             true,
	"manifest":        true,
	"max":             true,
	"maxlength":       true,
	"media":           true,
	"method":          true,
	"min":             true,
	"multiple":        true,
	"name":            true,
	"novalidate":      true,
	"open":            true,
	"optimum":         true,
	"pattern":         true,
	"ping":            true,
	"placeholder":     true,
	"poster":          true,
	// "preload":         true,
	"pubdate":    true,
	"radiogroup": true,
	"readonly":   true,
	"rel":        true,
	"required":   true,
	"reversed":   true,
	"rows":       true,
	"rowspan":    true,
	"sandbox":    true,
	"spellcheck": true,
	"scope":      true,
	// "scoped":          true,
	// "seamless":        true,
	"selected": true,
	"shape":    true,
	"size":     true,
	"sizes":    true,
	"span":     true,
	"src":      true,
	// "srcdoc":          true,
	"srclang": true,
	"start":   true,
	// "step":            true,
	"style":   true,
	"summary": true,
	// "tabindex":        true,
	// "target":          true,
	"title": true,
	"type":  true,
	// "usemap":          true,
	"value": true,
	"width": true,
	// "wrap":            true,

	// other allowed attributes
	"align":        true,
	"alink":        true,
	"background":   true,
	"cellpadding":  true,
	"cellspacing":  true,
	"char":         true,
	"clear":        true,
	"compact":      true,
	"frameborder":  true,
	"frame":        true,
	"hspace":       true,
	"marginheight": true,
	"noshade":      true,
	"nowrap":       true,
	"rules":        true,
	"scrolling":    true,
	"valign":       true,
}

// Based on list from Wikipedia's URI scheme
// http://en.wikipedia.org/wiki/URI_scheme
var acceptableUriSchemes = map[string]bool{
	"aim":      true,
	"apt":      true,
	"bitcoin":  true,
	"callto":   true,
	"cvs":      true,
	"facetime": true,
	"feed":     true,
	"ftp":      true,
	"git":      true,
	"gopher":   true,
	"gtalk":    true,
	"http":     true,
	"https":    true,
	"imap":     true,
	"irc":      true,
	"itms":     true,
	"jabber":   true,
	"magnet":   true,
	"mailto":   true,
	"mms":      true,
	"msnim":    true,
	"news":     true,
	"nntp":     true,
	"rtmp":     true,
	"rtsp":     true,
	"sftp":     true,
	"skype":    true,
	"svn":      true,
	"ymsgr":    true,
}
<|endoftext|>


<commit_before>package node

import (
	"encoding/json"
	"testing"
)

func TestPacketMarshalling(t *testing.T) {
	sk, _ := NewECDSAKey()
	m := Packet{sk.PublicKey().Hash(), 3, "test"}
	b, err := json.Marshal(m)
	if err != nil {
		t.Fatal(err)
	}
	var m2 Packet
	err = json.Unmarshal(b, &m2)
	if err != nil {
		t.Fatal(err)
	}
	if m2 != m {
		t.Fatalf("Different packets? %v != %v", m2, m)
	}
}
<commit_msg>Make sure Packet.String() doesn't crash<commit_after>package node

import (
	"encoding/json"
	"testing"
)

func TestPacketMarshalling(t *testing.T) {
	sk, _ := NewECDSAKey()
	m := Packet{sk.PublicKey().Hash(), 3, "test"}
	b, err := json.Marshal(m)
	if err != nil {
		t.Fatal(err)
	}
	var m2 Packet
	err = json.Unmarshal(b, &m2)
	if err != nil {
		t.Fatal(err)
	}
	if m2 != m {
		t.Fatalf("Different packets? %v != %v", m2, m)
	}
	_ = m.String()
}
<|endoftext|>


<commit_before>// Copyright 2017-present Andrea Funtò. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package openstack

// Authentication contains the identity entity used to
// authenticate users against a Keystone instance.
type Authentication struct {
	Identity *Identity `json:"identity,omitempty"`
}

type Identity struct {
	Methods  *[]string `json:"methods,omitempty"`
	Password *Password `json:"password,omitempty"`
}

type Scope struct {
	Project *Project `json:"project,omitempty"`
	Domain  *Domain  `json:"domain,omitempty"` // either one or the other: if both, BadRequest!
}

type Project struct {
	Id *string `json:"id,omitemty"`
}
type Password struct {
	User User `json:"user,omitempty"`
}

type User struct {
	Id                *string `json:"id,omitempty"`
	Name              *string `json:"name,omitempty"`
	Domain            *Domain `json:"domain,omitempty"`
	Password          *string `json:"password,omitempty"`
	PasswordExpiresAt *string `json:"password_expires_at,omitempty"`
}

type Domain struct {
	Id   *string `json:"id,omitempty"`
	Name *string `json:"name,omitempty"`
}

type Token struct {
	IssuedAt  *string   `json:"issued_at,omitempty"`
	ExpiresAt *string   `json:"expires_at,omitempty"`
	User      *User     `json:"user,omitempty"`
	Methods   *[]string `json:"methods,omitempty"`
	AuditIds  *[]string `json:"audit_ids,omitempty"`
}
<commit_msg>Added Roles to Token struct<commit_after>// Copyright 2017-present Andrea Funtò. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package openstack

// Authentication contains the identity entity used to
// authenticate users against a Keystone instance.
type Authentication struct {
	Identity *Identity `json:"identity,omitempty"`
}

type Identity struct {
	Methods  *[]string `json:"methods,omitempty"`
	Password *Password `json:"password,omitempty"`
}

type Scope struct {
	Project *Project `json:"project,omitempty"`
	Domain  *Domain  `json:"domain,omitempty"` // either one or the other: if both, BadRequest!
}

type Project struct {
	ID     *string `json:"id,omitemty"`
	Name   *string `json:"name,omitempty"`
	Domain *Domain `json:"domain,omitempty"`
}
type Password struct {
	User User `json:"user,omitempty"`
}

type User struct {
	ID                *string `json:"id,omitempty"`
	Name              *string `json:"name,omitempty"`
	Domain            *Domain `json:"domain,omitempty"`
	Password          *string `json:"password,omitempty"`
	PasswordExpiresAt *string `json:"password_expires_at,omitempty"`
}

type Domain struct {
	ID   *string `json:"id,omitempty"`
	Name *string `json:"name,omitempty"`
}

type Role struct {
	ID   *string `json:"id,omitempty"`
	Name *string `json:"name,omitempty"`
}

type Token struct {
	IssuedAt  *string   `json:"issued_at,omitempty"`
	ExpiresAt *string   `json:"expires_at,omitempty"`
	User      *User     `json:"user,omitempty"`
	Roles     *[]Role   `json:"roles,omitempty"`
	Methods   *[]string `json:"methods,omitempty"`
	AuditIds  *[]string `json:"audit_ids,omitempty"`
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interp

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"syscall"
	"time"

	"mvdan.cc/sh/v3/expand"
)

// FromModuleContext returns the ModuleCtx value stored in ctx, if any.
func FromModuleContext(ctx context.Context) (ModuleCtx, bool) {
	mc, ok := ctx.Value(moduleCtxKey{}).(ModuleCtx)
	return mc, ok
}

type moduleCtxKey struct{}

// ModuleCtx is the data passed to all the module functions via a context value.
// It contains some of the current state of the Runner, as well as some fields
// necessary to implement some of the modules.
type ModuleCtx struct {
	Env         expand.Environ
	Dir         string
	Stdin       io.Reader
	Stdout      io.Writer
	Stderr      io.Writer
	KillTimeout time.Duration
}

// UnixPath fixes absolute unix paths on Windows, for example converting
// "C:\\CurDir\\dev\\null" to "/dev/null".
func (mc ModuleCtx) UnixPath(path string) string {
	if runtime.GOOS != "windows" {
		return path
	}
	path = strings.TrimPrefix(path, mc.Dir)
	return strings.Replace(path, `\`, `/`, -1)
}

// ExecModule is the module responsible for executing a program. It is
// executed for all CallExpr nodes where the first argument is neither a
// declared function nor a builtin.
//
// Note that the name is included as the first argument. If path is an
// empty string, it means that the executable did not exist or was not
// found in $PATH.
//
// Use a return error of type ExitStatus to set the exit status. A nil error has
// the same effect as ExitStatus(0). If the error is of any other type, the
// interpreter will come to a stop.
type ExecModule = func(ctx context.Context, path string, args []string) error

func DefaultExec(ctx context.Context, path string, args []string) error {
	mc, _ := FromModuleContext(ctx)
	if path == "" {
		fmt.Fprintf(mc.Stderr, "%q: executable file not found in $PATH\n", args[0])
		return ExitStatus(127)
	}
	cmd := exec.Cmd{
		Path:   path,
		Args:   args,
		Env:    execEnv(mc.Env),
		Dir:    mc.Dir,
		Stdin:  mc.Stdin,
		Stdout: mc.Stdout,
		Stderr: mc.Stderr,
	}

	err := cmd.Start()
	if err == nil {
		if done := ctx.Done(); done != nil {
			go func() {
				<-done

				if mc.KillTimeout <= 0 || runtime.GOOS == "windows" {
					_ = cmd.Process.Signal(os.Kill)
					return
				}

				// TODO: don't temporarily leak this goroutine
				// if the program stops itself with the
				// interrupt.
				go func() {
					time.Sleep(mc.KillTimeout)
					_ = cmd.Process.Signal(os.Kill)
				}()
				_ = cmd.Process.Signal(os.Interrupt)
			}()
		}

		err = cmd.Wait()
	}

	switch x := err.(type) {
	case *exec.ExitError:
		// started, but errored - default to 1 if OS
		// doesn't have exit statuses
		if status, ok := x.Sys().(syscall.WaitStatus); ok {
			if status.Signaled() && ctx.Err() != nil {
				return ctx.Err()
			}
			return ExitStatus(status.ExitStatus())
		}
		return ExitStatus(1)
	case *exec.Error:
		// did not start
		fmt.Fprintf(mc.Stderr, "%v\n", err)
		return ExitStatus(127)
	default:
		return err
	}
}

func ExecBuiltin(name string, fn func(ModuleCtx, []string) error) func(ExecModule) ExecModule {
	return func(next ExecModule) ExecModule {
		return func(ctx context.Context, path string, args []string) error {
			if args[0] == name {
				mc, _ := FromModuleContext(ctx)
				return fn(mc, args[1:])
			}
			return next(ctx, path, args)
		}
	}
}

// OpenModule is the module responsible for opening a file. It is
// executed for all files that are opened directly by the shell, such as
// in redirects. Files opened by executed programs are not included.
//
// The path parameter is absolute and has been cleaned.
//
// Use a return error of type *os.PathError to have the error printed to
// stderr and the exit status set to 1. If the error is of any other type, the
// interpreter will come to a stop.
//
// TODO: What about stat calls? They are used heavily in the builtin
// test expressions, and also when doing a cd. Should they have a
// separate module?
type OpenModule = func(ctx context.Context, path string, flag int, perm os.FileMode) (io.ReadWriteCloser, error)

func DefaultOpen(ctx context.Context, path string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
	return os.OpenFile(path, flag, perm)
}

func OpenDevImpls(next OpenModule) OpenModule {
	return func(ctx context.Context, path string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
		mc, _ := FromModuleContext(ctx)
		switch mc.UnixPath(path) {
		case "/dev/null":
			return devNull{}, nil
		}
		return next(ctx, path, flag, perm)
	}
}

var _ io.ReadWriteCloser = devNull{}

type devNull struct{}

func (devNull) Read(p []byte) (int, error)  { return 0, io.EOF }
func (devNull) Write(p []byte) (int, error) { return len(p), nil }
func (devNull) Close() error                { return nil }
<commit_msg>interp: gofmt<commit_after>// Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interp

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"syscall"
	"time"

	"mvdan.cc/sh/v3/expand"
)

// FromModuleContext returns the ModuleCtx value stored in ctx, if any.
func FromModuleContext(ctx context.Context) (ModuleCtx, bool) {
	mc, ok := ctx.Value(moduleCtxKey{}).(ModuleCtx)
	return mc, ok
}

type moduleCtxKey struct{}

// ModuleCtx is the data passed to all the module functions via a context value.
// It contains some of the current state of the Runner, as well as some fields
// necessary to implement some of the modules.
type ModuleCtx struct {
	Env         expand.Environ
	Dir         string
	Stdin       io.Reader
	Stdout      io.Writer
	Stderr      io.Writer
	KillTimeout time.Duration
}

// UnixPath fixes absolute unix paths on Windows, for example converting
// "C:\\CurDir\\dev\\null" to "/dev/null".
func (mc ModuleCtx) UnixPath(path string) string {
	if runtime.GOOS != "windows" {
		return path
	}
	path = strings.TrimPrefix(path, mc.Dir)
	return strings.Replace(path, `\`, `/`, -1)
}

// ExecModule is the module responsible for executing a program. It is
// executed for all CallExpr nodes where the first argument is neither a
// declared function nor a builtin.
//
// Note that the name is included as the first argument. If path is an
// empty string, it means that the executable did not exist or was not
// found in $PATH.
//
// Use a return error of type ExitStatus to set the exit status. A nil error has
// the same effect as ExitStatus(0). If the error is of any other type, the
// interpreter will come to a stop.
type ExecModule = func(ctx context.Context, path string, args []string) error

func DefaultExec(ctx context.Context, path string, args []string) error {
	mc, _ := FromModuleContext(ctx)
	if path == "" {
		fmt.Fprintf(mc.Stderr, "%q: executable file not found in $PATH\n", args[0])
		return ExitStatus(127)
	}
	cmd := exec.Cmd{
		Path:   path,
		Args:   args,
		Env:    execEnv(mc.Env),
		Dir:    mc.Dir,
		Stdin:  mc.Stdin,
		Stdout: mc.Stdout,
		Stderr: mc.Stderr,
	}

	err := cmd.Start()
	if err == nil {
		if done := ctx.Done(); done != nil {
			go func() {
				<-done

				if mc.KillTimeout <= 0 || runtime.GOOS == "windows" {
					_ = cmd.Process.Signal(os.Kill)
					return
				}

				// TODO: don't temporarily leak this goroutine
				// if the program stops itself with the
				// interrupt.
				go func() {
					time.Sleep(mc.KillTimeout)
					_ = cmd.Process.Signal(os.Kill)
				}()
				_ = cmd.Process.Signal(os.Interrupt)
			}()
		}

		err = cmd.Wait()
	}

	switch x := err.(type) {
	case *exec.ExitError:
		// started, but errored - default to 1 if OS
		// doesn't have exit statuses
		if status, ok := x.Sys().(syscall.WaitStatus); ok {
			if status.Signaled() && ctx.Err() != nil {
				return ctx.Err()
			}
			return ExitStatus(status.ExitStatus())
		}
		return ExitStatus(1)
	case *exec.Error:
		// did not start
		fmt.Fprintf(mc.Stderr, "%v\n", err)
		return ExitStatus(127)
	default:
		return err
	}
}

func ExecBuiltin(name string, fn func(ModuleCtx, []string) error) func(ExecModule) ExecModule {
	return func(next ExecModule) ExecModule {
		return func(ctx context.Context, path string, args []string) error {
			if args[0] == name {
				mc, _ := FromModuleContext(ctx)
				return fn(mc, args[1:])
			}
			return next(ctx, path, args)
		}
	}
}

// OpenModule is the module responsible for opening a file. It is
// executed for all files that are opened directly by the shell, such as
// in redirects. Files opened by executed programs are not included.
//
// The path parameter is absolute and has been cleaned.
//
// Use a return error of type *os.PathError to have the error printed to
// stderr and the exit status set to 1. If the error is of any other type, the
// interpreter will come to a stop.
//
// TODO: What about stat calls? They are used heavily in the builtin
// test expressions, and also when doing a cd. Should they have a
// separate module?
type OpenModule = func(ctx context.Context, path string, flag int, perm os.FileMode) (io.ReadWriteCloser, error)

func DefaultOpen(ctx context.Context, path string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
	return os.OpenFile(path, flag, perm)
}

func OpenDevImpls(next OpenModule) OpenModule {
	return func(ctx context.Context, path string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
		mc, _ := FromModuleContext(ctx)
		switch mc.UnixPath(path) {
		case "/dev/null":
			return devNull{}, nil
		}
		return next(ctx, path, flag, perm)
	}
}

var _ io.ReadWriteCloser = devNull{}

type devNull struct{}

func (devNull) Read(p []byte) (int, error)  { return 0, io.EOF }
func (devNull) Write(p []byte) (int, error) { return len(p), nil }
func (devNull) Close() error                { return nil }
<|endoftext|>


<commit_before>package gcs

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"path"

	"cloud.google.com/go/storage"
	"github.com/estafette/estafette-ci-api/config"
	contracts "github.com/estafette/estafette-ci-contracts"
	"github.com/opentracing/opentracing-go"
)

// CloudStorageClient is the interface for connecting to google cloud storage
type CloudStorageClient interface {
	InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error)
	InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error)
	GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog) (updatedBuildLog contracts.BuildLog, err error)
	GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog) (updatedReleaseLog contracts.ReleaseLog, err error)
}

type cloudStorageClientImpl struct {
	client *storage.Client
	config *config.CloudStorageConfig
}

// NewCloudStorageClient returns new CloudStorageClient
func NewCloudStorageClient(config *config.CloudStorageConfig) (CloudStorageClient, error) {

	if config == nil {
		return &cloudStorageClientImpl{
			config: config,
		}, nil
	}

	ctx := context.Background()

	storageClient, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}

	return &cloudStorageClientImpl{
		client: storageClient,
		config: config,
	}, nil
}

func (impl *cloudStorageClientImpl) InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::InsertBuildLog")
	defer span.Finish()

	logPath := impl.getBuildLogPath(buildLog)

	return impl.insertLog(ctx, logPath, buildLog.Steps)
}

func (impl *cloudStorageClientImpl) InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::InsertReleaseLog")
	defer span.Finish()

	logPath := impl.getReleaseLogPath(releaseLog)

	return impl.insertLog(ctx, logPath, releaseLog.Steps)
}

func (impl *cloudStorageClientImpl) GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog) (updatedBuildLog contracts.BuildLog, err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::GetPipelineBuildLogs")
	defer span.Finish()

	logPath := impl.getBuildLogPath(buildLog)

	steps, err := impl.getLog(ctx, logPath)
	if err != nil {
		return buildLog, err
	}

	updatedBuildLog = buildLog
	updatedBuildLog.Steps = steps

	return updatedBuildLog, nil
}

func (impl *cloudStorageClientImpl) GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog) (updatedReleaseLog contracts.ReleaseLog, err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::GetPipelineReleaseLogs")
	defer span.Finish()

	logPath := impl.getReleaseLogPath(releaseLog)

	steps, err := impl.getLog(ctx, logPath)
	if err != nil {
		return releaseLog, err
	}

	updatedReleaseLog = releaseLog
	updatedReleaseLog.Steps = steps

	return updatedReleaseLog, nil

}

func (impl *cloudStorageClientImpl) insertLog(ctx context.Context, path string, steps []*contracts.BuildLogStep) (err error) {

	bucket := impl.client.Bucket(impl.config.Bucket)

	// marshal json
	jsonBytes, err := json.Marshal(steps)
	if err != nil {
		return err
	}

	// create writer for cloud storage object
	logObject := bucket.Object(path)

	// don't allow overwrites, return when file already exists
	_, err = logObject.Attrs(ctx)
	if err == nil {
		// log file already exists, return
		return nil
	}
	if err != nil && err != storage.ErrObjectNotExist {
		// some other error happened, return it
		return err
	}

	// object doesn't exist, okay to write it
	writer := logObject.NewWriter(ctx)
	if writer == nil {
		return fmt.Errorf("Writer for logobject %v is nil", path)
	}

	// write compressed bytes
	gz, err := gzip.NewWriterLevel(writer, gzip.BestSpeed)
	if err != nil {
		_ = writer.Close()
		return err
	}
	_, err = gz.Write(jsonBytes)
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = gz.Close()
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}

func (impl *cloudStorageClientImpl) getLog(ctx context.Context, path string) (steps []*contracts.BuildLogStep, err error) {

	bucket := impl.client.Bucket(impl.config.Bucket)

	// create reader for cloud storage object
	logObject := bucket.Object(path).ReadCompressed(true)
	reader, err := logObject.NewReader(ctx)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	// read compressed bytes
	gzr, err := gzip.NewReader(reader)
	if err != nil {
		return nil, err
	}
	defer gzr.Close()

	// unmarshal json
	decoder := json.NewDecoder(gzr)
	err = decoder.Decode(&steps)
	if err != nil {
		return nil, err
	}

	return
}

func (impl *cloudStorageClientImpl) getBuildLogPath(buildLog contracts.BuildLog) (logPath string) {

	logPath = path.Join(impl.config.LogsDirectory, buildLog.RepoSource, buildLog.RepoOwner, buildLog.RepoName, "builds", fmt.Sprintf("%v.log", buildLog.ID))

	return logPath
}

func (impl *cloudStorageClientImpl) getReleaseLogPath(releaseLog contracts.ReleaseLog) (logPath string) {

	logPath = path.Join(impl.config.LogsDirectory, releaseLog.RepoSource, releaseLog.RepoOwner, releaseLog.RepoName, "releases", fmt.Sprintf("%v.log", releaseLog.ID))

	return logPath
}
<commit_msg>use ioutil.ReadAll to read log file from cloud storage<commit_after>package gcs

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path"

	"cloud.google.com/go/storage"
	"github.com/estafette/estafette-ci-api/config"
	contracts "github.com/estafette/estafette-ci-contracts"
	"github.com/opentracing/opentracing-go"
)

// CloudStorageClient is the interface for connecting to google cloud storage
type CloudStorageClient interface {
	InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error)
	InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error)
	GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog) (updatedBuildLog contracts.BuildLog, err error)
	GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog) (updatedReleaseLog contracts.ReleaseLog, err error)
}

type cloudStorageClientImpl struct {
	client *storage.Client
	config *config.CloudStorageConfig
}

// NewCloudStorageClient returns new CloudStorageClient
func NewCloudStorageClient(config *config.CloudStorageConfig) (CloudStorageClient, error) {

	if config == nil {
		return &cloudStorageClientImpl{
			config: config,
		}, nil
	}

	ctx := context.Background()

	storageClient, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}

	return &cloudStorageClientImpl{
		client: storageClient,
		config: config,
	}, nil
}

func (impl *cloudStorageClientImpl) InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::InsertBuildLog")
	defer span.Finish()

	logPath := impl.getBuildLogPath(buildLog)

	return impl.insertLog(ctx, logPath, buildLog.Steps)
}

func (impl *cloudStorageClientImpl) InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::InsertReleaseLog")
	defer span.Finish()

	logPath := impl.getReleaseLogPath(releaseLog)

	return impl.insertLog(ctx, logPath, releaseLog.Steps)
}

func (impl *cloudStorageClientImpl) GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog) (updatedBuildLog contracts.BuildLog, err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::GetPipelineBuildLogs")
	defer span.Finish()

	logPath := impl.getBuildLogPath(buildLog)

	steps, err := impl.getLog(ctx, logPath)
	if err != nil {
		return buildLog, err
	}

	updatedBuildLog = buildLog
	updatedBuildLog.Steps = steps

	return updatedBuildLog, nil
}

func (impl *cloudStorageClientImpl) GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog) (updatedReleaseLog contracts.ReleaseLog, err error) {

	span, ctx := opentracing.StartSpanFromContext(ctx, "CloudStorageClient::GetPipelineReleaseLogs")
	defer span.Finish()

	logPath := impl.getReleaseLogPath(releaseLog)

	steps, err := impl.getLog(ctx, logPath)
	if err != nil {
		return releaseLog, err
	}

	updatedReleaseLog = releaseLog
	updatedReleaseLog.Steps = steps

	return updatedReleaseLog, nil

}

func (impl *cloudStorageClientImpl) insertLog(ctx context.Context, path string, steps []*contracts.BuildLogStep) (err error) {

	bucket := impl.client.Bucket(impl.config.Bucket)

	// marshal json
	jsonBytes, err := json.Marshal(steps)
	if err != nil {
		return err
	}

	// create writer for cloud storage object
	logObject := bucket.Object(path)

	// don't allow overwrites, return when file already exists
	_, err = logObject.Attrs(ctx)
	if err == nil {
		// log file already exists, return
		return nil
	}
	if err != nil && err != storage.ErrObjectNotExist {
		// some other error happened, return it
		return err
	}

	// object doesn't exist, okay to write it
	writer := logObject.NewWriter(ctx)
	if writer == nil {
		return fmt.Errorf("Writer for logobject %v is nil", path)
	}

	// write compressed bytes
	gz, err := gzip.NewWriterLevel(writer, gzip.BestSpeed)
	if err != nil {
		_ = writer.Close()
		return err
	}
	_, err = gz.Write(jsonBytes)
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = gz.Close()
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}

func (impl *cloudStorageClientImpl) getLog(ctx context.Context, path string) (steps []*contracts.BuildLogStep, err error) {

	bucket := impl.client.Bucket(impl.config.Bucket)

	// create reader for cloud storage object
	logObject := bucket.Object(path).ReadCompressed(true)
	reader, err := logObject.NewReader(ctx)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	// read compressed bytes
	gzr, err := gzip.NewReader(reader)
	if err != nil {
		return nil, err
	}
	defer gzr.Close()

	// read entire file
	bytes, err := ioutil.ReadAll(gzr)
	if err != nil {
		return nil, err
	}

	// unmarshal json
	err = json.Unmarshal(bytes, &steps)
	if err != nil {
		return nil, err
	}

	return
}

func (impl *cloudStorageClientImpl) getBuildLogPath(buildLog contracts.BuildLog) (logPath string) {

	logPath = path.Join(impl.config.LogsDirectory, buildLog.RepoSource, buildLog.RepoOwner, buildLog.RepoName, "builds", fmt.Sprintf("%v.log", buildLog.ID))

	return logPath
}

func (impl *cloudStorageClientImpl) getReleaseLogPath(releaseLog contracts.ReleaseLog) (logPath string) {

	logPath = path.Join(impl.config.LogsDirectory, releaseLog.RepoSource, releaseLog.RepoOwner, releaseLog.RepoName, "releases", fmt.Sprintf("%v.log", releaseLog.ID))

	return logPath
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io"

	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/mutable"
	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// Safe for concurrent access.
type ObjectSyncer interface {
	// Given an object record and content that was originally derived from that
	// object's contents (and potentially modified):
	//
	// *   If the content has not been modified, return a nil read lease and a
	//     nil new object.
	//
	// *   Otherwise, write out a new generation in the bucket (failing with
	//     *gcs.PreconditionError if the source generation is no longer current)
	//     and return a read lease for that object's contents.
	//
	// In the second case, the mutable.Content is destroyed. Otherwise, including
	// when this function fails, it is guaranteed to still be valid.
	SyncObject(
		ctx context.Context,
		srcObject *gcs.Object,
		content mutable.Content) (rl lease.ReadLease, o *gcs.Object, err error)
}

// Create an object syncer that syncs into the supplied bucket.
//
// When the source object has been changed only by appending, and the source
// object's size is at least appendThreshold, we will "append" to it by writing
// out a temporary blob and composing it with the source object.
//
// Temporary blobs have names beginning with tmpObjectPrefix. We make an effort
// to delete them, but if we are interrupted for some reason we may not be able
// to do so. Therefore the user should arrange for garbage collection.
func NewObjectSyncer(
	appendThreshold int64,
	tmpObjectPrefix string,
	bucket gcs.Bucket) (os ObjectSyncer) {
	// Create the append object creator.
	appendCreator := newAppendObjectCreator(
		tmpObjectPrefix,
		bucket)

	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// objectSyncer
////////////////////////////////////////////////////////////////////////

// An implementation detail of objectSyncer. See notes on
// newObjectSyncer.
type objectCreator interface {
	Create(
		ctx context.Context,
		srcObject *gcs.Object,
		r io.Reader) (o *gcs.Object, err error)
}

// Create an object syncer that stats the mutable content to see if it's dirty
// before calling through to one of two object creators if the content is dirty:
//
// *   fullCreator accepts the source object and the full contents with which it
//     should be overwritten.
//
// *   appendCreator accepts the source object and the contents that should be
//     "appended" to it.
//
// appendThreshold controls the source object length at which we consider it
// worthwhile to make the append optimization. It should be set to a value on
// the order of the bandwidth to GCS times three times the round trip latency
// to GCS (for a small create, a compose, and a delete).
func newObjectSyncer(
	appendThreshold int64,
	fullCreator objectCreator,
	appendCreator objectCreator) (os ObjectSyncer) {
	os = &objectSyncer{
		appendThreshold: appendThreshold,
		fullCreator:     fullCreator,
		appendCreator:   appendCreator,
	}

	return
}

type objectSyncer struct {
	appendThreshold int64
	fullCreator     objectCreator
	appendCreator   objectCreator
}

func (os *objectSyncer) SyncObject(
	ctx context.Context,
	srcObject *gcs.Object,
	content mutable.Content) (rl lease.ReadLease, o *gcs.Object, err error) {
	// Stat the content.
	sr, err := content.Stat(ctx)
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	// Make sure the dirty threshold makes sense.
	srcSize := int64(srcObject.Size)
	if sr.DirtyThreshold > srcSize {
		err = fmt.Errorf(
			"Stat returned weird DirtyThreshold field: %d vs. %d",
			sr.DirtyThreshold,
			srcObject.Size)

		return
	}

	// If the content hasn't been dirtied (i.e. it is the same size as the source
	// object, and no bytes within the source object have been dirtied), we're
	// done.
	if sr.Size == srcSize && sr.DirtyThreshold == srcSize {
		return
	}

	// Otherwise, we need to create a new generation. If the source object is
	// long enough, hasn't been dirtied, and has a low enough component count,
	// then we can make the optimization of not rewriting its contents.
	if srcSize >= os.appendThreshold &&
		sr.DirtyThreshold == srcSize &&
		srcObject.ComponentCount < gcs.MaxComponentCount {
		o, err = os.appendCreator.Create(
			ctx,
			srcObject,
			&mutableContentReader{
				Ctx:     ctx,
				Content: content,
				Offset:  srcSize,
			})
	} else {
		o, err = os.fullCreator.Create(
			ctx,
			srcObject,
			&mutableContentReader{
				Ctx:     ctx,
				Content: content,
			})
	}

	// Deal with errors.
	if err != nil {
		// Special case: don't mess with precondition errors.
		if _, ok := err.(*gcs.PreconditionError); ok {
			return
		}

		err = fmt.Errorf("Create: %v", err)
		return
	}

	// Yank out the contents.
	rl = content.Release().Downgrade()

	return
}

////////////////////////////////////////////////////////////////////////
// mutableContentReader
////////////////////////////////////////////////////////////////////////

// An io.Reader that wraps a mutable.Content object, reading starting from a
// base offset.
type mutableContentReader struct {
	Ctx     context.Context
	Content mutable.Content
	Offset  int64
}

func (mcr *mutableContentReader) Read(p []byte) (n int, err error) {
	n, err = mcr.Content.ReadAt(mcr.Ctx, p, mcr.Offset)
	mcr.Offset += int64(n)
	return
}
<commit_msg>NewObjectSyncer<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io"

	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/mutable"
	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// Safe for concurrent access.
type ObjectSyncer interface {
	// Given an object record and content that was originally derived from that
	// object's contents (and potentially modified):
	//
	// *   If the content has not been modified, return a nil read lease and a
	//     nil new object.
	//
	// *   Otherwise, write out a new generation in the bucket (failing with
	//     *gcs.PreconditionError if the source generation is no longer current)
	//     and return a read lease for that object's contents.
	//
	// In the second case, the mutable.Content is destroyed. Otherwise, including
	// when this function fails, it is guaranteed to still be valid.
	SyncObject(
		ctx context.Context,
		srcObject *gcs.Object,
		content mutable.Content) (rl lease.ReadLease, o *gcs.Object, err error)
}

// Create an object syncer that syncs into the supplied bucket.
//
// When the source object has been changed only by appending, and the source
// object's size is at least appendThreshold, we will "append" to it by writing
// out a temporary blob and composing it with the source object.
//
// Temporary blobs have names beginning with tmpObjectPrefix. We make an effort
// to delete them, but if we are interrupted for some reason we may not be able
// to do so. Therefore the user should arrange for garbage collection.
func NewObjectSyncer(
	appendThreshold int64,
	tmpObjectPrefix string,
	bucket gcs.Bucket) (os ObjectSyncer) {
	// Create the object creators.
	fullCreator := &fullObjectCreator{
		bucket: bucket,
	}

	appendCreator := newAppendObjectCreator(
		tmpObjectPrefix,
		bucket)

	// And the object syncer.
	os = newObjectSyncer(appendThreshold, fullCreator, appendCreator)

	return
}

////////////////////////////////////////////////////////////////////////
// fullObjectCreator
////////////////////////////////////////////////////////////////////////

type fullObjectCreator struct {
	bucket gcs.Bucket
}

func (oc *fullObjectCreator) Create(
	ctx context.Context,
	srcObject *gcs.Object,
	r io.Reader) (o *gcs.Object, err error) {
	req := &gcs.CreateObjectRequest{
		Name: srcObject.Name,
		GenerationPrecondition: &srcObject.Generation,
		Contents:               r,
	}

	o, err = oc.bucket.CreateObject(ctx, req)
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// objectSyncer
////////////////////////////////////////////////////////////////////////

// An implementation detail of objectSyncer. See notes on
// newObjectSyncer.
type objectCreator interface {
	Create(
		ctx context.Context,
		srcObject *gcs.Object,
		r io.Reader) (o *gcs.Object, err error)
}

// Create an object syncer that stats the mutable content to see if it's dirty
// before calling through to one of two object creators if the content is dirty:
//
// *   fullCreator accepts the source object and the full contents with which it
//     should be overwritten.
//
// *   appendCreator accepts the source object and the contents that should be
//     "appended" to it.
//
// appendThreshold controls the source object length at which we consider it
// worthwhile to make the append optimization. It should be set to a value on
// the order of the bandwidth to GCS times three times the round trip latency
// to GCS (for a small create, a compose, and a delete).
func newObjectSyncer(
	appendThreshold int64,
	fullCreator objectCreator,
	appendCreator objectCreator) (os ObjectSyncer) {
	os = &objectSyncer{
		appendThreshold: appendThreshold,
		fullCreator:     fullCreator,
		appendCreator:   appendCreator,
	}

	return
}

type objectSyncer struct {
	appendThreshold int64
	fullCreator     objectCreator
	appendCreator   objectCreator
}

func (os *objectSyncer) SyncObject(
	ctx context.Context,
	srcObject *gcs.Object,
	content mutable.Content) (rl lease.ReadLease, o *gcs.Object, err error) {
	// Stat the content.
	sr, err := content.Stat(ctx)
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	// Make sure the dirty threshold makes sense.
	srcSize := int64(srcObject.Size)
	if sr.DirtyThreshold > srcSize {
		err = fmt.Errorf(
			"Stat returned weird DirtyThreshold field: %d vs. %d",
			sr.DirtyThreshold,
			srcObject.Size)

		return
	}

	// If the content hasn't been dirtied (i.e. it is the same size as the source
	// object, and no bytes within the source object have been dirtied), we're
	// done.
	if sr.Size == srcSize && sr.DirtyThreshold == srcSize {
		return
	}

	// Otherwise, we need to create a new generation. If the source object is
	// long enough, hasn't been dirtied, and has a low enough component count,
	// then we can make the optimization of not rewriting its contents.
	if srcSize >= os.appendThreshold &&
		sr.DirtyThreshold == srcSize &&
		srcObject.ComponentCount < gcs.MaxComponentCount {
		o, err = os.appendCreator.Create(
			ctx,
			srcObject,
			&mutableContentReader{
				Ctx:     ctx,
				Content: content,
				Offset:  srcSize,
			})
	} else {
		o, err = os.fullCreator.Create(
			ctx,
			srcObject,
			&mutableContentReader{
				Ctx:     ctx,
				Content: content,
			})
	}

	// Deal with errors.
	if err != nil {
		// Special case: don't mess with precondition errors.
		if _, ok := err.(*gcs.PreconditionError); ok {
			return
		}

		err = fmt.Errorf("Create: %v", err)
		return
	}

	// Yank out the contents.
	rl = content.Release().Downgrade()

	return
}

////////////////////////////////////////////////////////////////////////
// mutableContentReader
////////////////////////////////////////////////////////////////////////

// An io.Reader that wraps a mutable.Content object, reading starting from a
// base offset.
type mutableContentReader struct {
	Ctx     context.Context
	Content mutable.Content
	Offset  int64
}

func (mcr *mutableContentReader) Read(p []byte) (n int, err error) {
	n, err = mcr.Content.ReadAt(mcr.Ctx, p, mcr.Offset)
	mcr.Offset += int64(n)
	return
}
<|endoftext|>