<commit_before>package types

// transaction.go defines the transaction type and all of the sub-fields of the
// transaction, as well as providing helper functions for working with
// transactions. The various IDs are designed such that, in a legal blockchain,
// it is cryptographically unlikely that any two objects would share an id.

import (
	"errors"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
)

const (
	SpecifierLen = 16

	// UnlockHashChecksumSize is the size of the checksum used to verify
	// human-readable addresses. It is not a crypytographically secure
	// checksum, it's merely intended to prevent typos. 6 is chosen because it
	// brings the total size of the address to 38 bytes, leaving 2 bytes for
	// potential version additions in the future.
	UnlockHashChecksumSize = 6
)

// These Specifiers are used internally when calculating a type's ID. See
// Specifier for more details.
var (
	SpecifierMinerPayout          = Specifier{'m', 'i', 'n', 'e', 'r', ' ', 'p', 'a', 'y', 'o', 'u', 't'}
	SpecifierSiacoinInput         = Specifier{'s', 'i', 'a', 'c', 'o', 'i', 'n', ' ', 'i', 'n', 'p', 'u', 't'}
	SpecifierSiacoinOutput        = Specifier{'s', 'i', 'a', 'c', 'o', 'i', 'n', ' ', 'o', 'u', 't', 'p', 'u', 't'}
	SpecifierFileContract         = Specifier{'f', 'i', 'l', 'e', ' ', 'c', 'o', 'n', 't', 'r', 'a', 'c', 't'}
	SpecifierFileContractRevision = Specifier{'f', 'i', 'l', 'e', ' ', 'c', 'o', 'n', 't', 'r', 'a', 'c', 't', ' ', 'r', 'e'}
	SpecifierStorageProof         = Specifier{'s', 't', 'o', 'r', 'a', 'g', 'e', ' ', 'p', 'r', 'o', 'o', 'f'}
	SpecifierStorageProofOutput   = Specifier{'s', 't', 'o', 'r', 'a', 'g', 'e', ' ', 'p', 'r', 'o', 'o', 'f'}
	SpecifierSiafundInput         = Specifier{'s', 'i', 'a', 'f', 'u', 'n', 'd', ' ', 'i', 'n', 'p', 'u', 't'}
	SpecifierSiafundOutput        = Specifier{'s', 'i', 'a', 'f', 'u', 'n', 'd', ' ', 'o', 'u', 't', 'p', 'u', 't'}
	SpecifierClaimOutput          = Specifier{'c', 'l', 'a', 'i', 'm', ' ', 'o', 'u', 't', 'p', 'u', 't'}
	SpecifierMinerFee             = Specifier{'m', 'i', 'n', 'e', 'r', ' ', 'f', 'e', 'e'}

	ErrTransactionIDWrongLen = errors.New("input has wrong length to be an encoded transaction id")
)

type (
	// A Specifier is a fixed-length byte-array that serves two purposes. In
	// the wire protocol, they are used to identify a particular encoding
	// algorithm, signature algorithm, etc. This allows nodes to communicate on
	// their own terms; for example, to reduce bandwidth costs, a node might
	// only accept compressed messages.
	//
	// Internally, Specifiers are used to guarantee unique IDs. Various
	// consensus types have an associated ID, calculated by hashing the data
	// contained in the type. By prepending the data with Specifier, we can
	// guarantee that distinct types will never produce the same hash.
	Specifier [SpecifierLen]byte

	// IDs are used to refer to a type without revealing its contents. They
	// are constructed by hashing specific fields of the type, along with a
	// Specifier. While all of these types are hashes, defining type aliases
	// gives us type safety and makes the code more readable.
	TransactionID   crypto.Hash
	SiacoinOutputID crypto.Hash
	SiafundOutputID crypto.Hash
	FileContractID  crypto.Hash
	OutputID        crypto.Hash

	// A Transaction is an atomic component of a block. Transactions can contain
	// inputs and outputs, file contracts, storage proofs, and even arbitrary
	// data. They can also contain signatures to prove that a given party has
	// approved the transaction, or at least a particular subset of it.
	//
	// Transactions can depend on other previous transactions in the same block,
	// but transactions cannot spend outputs that they create or otherwise be
	// self-dependent.
	Transaction struct {
		SiacoinInputs         []SiacoinInput         `json:"siacoininputs"`
		SiacoinOutputs        []SiacoinOutput        `json:"siacoinoutputs"`
		FileContracts         []FileContract         `json:"filecontracts"`
		FileContractRevisions []FileContractRevision `json:"filecontractrevisions"`
		StorageProofs         []StorageProof         `json:"storageproofs"`
		SiafundInputs         []SiafundInput         `json:"siafundinputs"`
		SiafundOutputs        []SiafundOutput        `json:"siafundoutputs"`
		MinerFees             []Currency             `json:"minerfees"`
		ArbitraryData         [][]byte               `json:"arbitrarydata"`
		TransactionSignatures []TransactionSignature `json:"transactionsignatures"`
	}

	// A SiacoinInput consumes a SiacoinOutput and adds the siacoins to the set of
	// siacoins that can be spent in the transaction. The ParentID points to the
	// output that is getting consumed, and the UnlockConditions contain the rules
	// for spending the output. The UnlockConditions must match the UnlockHash of
	// the output.
	SiacoinInput struct {
		ParentID         SiacoinOutputID  `json:"parentid"`
		UnlockConditions UnlockConditions `json:"unlockconditions"`
	}

	// A SiacoinOutput holds a volume of siacoins. Outputs must be spent
	// atomically; that is, they must all be spent in the same transaction. The
	// UnlockHash is the hash of the UnlockConditions that must be fulfilled
	// in order to spend the output.
	SiacoinOutput struct {
		Value      Currency   `json:"value"`
		UnlockHash UnlockHash `json:"unlockhash"`
	}

	// A SiafundInput consumes a SiafundOutput and adds the siafunds to the set of
	// siafunds that can be spent in the transaction. The ParentID points to the
	// output that is getting consumed, and the UnlockConditions contain the rules
	// for spending the output. The UnlockConditions must match the UnlockHash of
	// the output.
	SiafundInput struct {
		ParentID         SiafundOutputID  `json:"parentid"`
		UnlockConditions UnlockConditions `json:"unlockconditions"`
		ClaimUnlockHash  UnlockHash       `json:"claimunlockhash"`
	}

	// A SiafundOutput holds a volume of siafunds. Outputs must be spent
	// atomically; that is, they must all be spent in the same transaction. The
	// UnlockHash is the hash of a set of UnlockConditions that must be fulfilled
	// in order to spend the output.
	//
	// When the SiafundOutput is spent, a SiacoinOutput is created, where:
	//
	//     SiacoinOutput.Value := (SiafundPool - ClaimStart) / 10,000
	//     SiacoinOutput.UnlockHash := SiafundOutput.ClaimUnlockHash
	//
	// When a SiafundOutput is put into a transaction, the ClaimStart must always
	// equal zero. While the transaction is being processed, the ClaimStart is set
	// to the value of the SiafundPool.
	SiafundOutput struct {
		Value      Currency   `json:"value"`
		UnlockHash UnlockHash `json:"unlockhash"`
		ClaimStart Currency   `json:"claimstart"`
	}

	// An UnlockHash is a specially constructed hash of the UnlockConditions type.
	// "Locked" values can be unlocked by providing the UnlockConditions that hash
	// to a given UnlockHash. See UnlockConditions.UnlockHash for details on how the
	// UnlockHash is constructed.
	UnlockHash crypto.Hash
)

// ID returns the id of a transaction, which is taken by marshalling all of the
// fields except for the signatures and taking the hash of the result.
func (t Transaction) ID() TransactionID {
	var txid TransactionID
	h := crypto.NewHash()
	enc := encoding.NewEncoder(h)

	for i := range t.SiacoinInputs {
		t.SiacoinInputs[i].MarshalSia(h)
	}
	encoding.WriteInt(h, len((t.SiacoinOutputs)))
	for i := range t.SiacoinOutputs {
		t.SiacoinOutputs[i].MarshalSia(h)
	}
	encoding.WriteInt(h, len((t.FileContracts)))
	for i := range t.FileContracts {
		enc.Encode(t.FileContracts[i])
	}
	encoding.WriteInt(h, len((t.FileContractRevisions)))
	for i := range t.FileContractRevisions {
		enc.Encode(t.FileContractRevisions[i])
	}
	encoding.WriteInt(h, len((t.StorageProofs)))
	for i := range t.StorageProofs {
		enc.Encode(t.StorageProofs[i])
	}
	encoding.WriteInt(h, len((t.SiafundInputs)))
	for i := range t.SiafundInputs {
		enc.Encode(t.SiafundInputs[i])
	}
	encoding.WriteInt(h, len((t.SiafundOutputs)))
	for i := range t.SiafundOutputs {
		t.SiafundOutputs[i].MarshalSia(h)
	}
	encoding.WriteInt(h, len((t.MinerFees)))
	for i := range t.MinerFees {
		t.MinerFees[i].MarshalSia(h)
	}
	encoding.WriteInt(h, len((t.ArbitraryData)))
	for i := range t.ArbitraryData {
		encoding.WritePrefix(h, t.ArbitraryData[i])
	}
	encoding.WriteInt(h, len((t.TransactionSignatures)))
	for i := range t.TransactionSignatures {
		t.TransactionSignatures[i].MarshalSia(h)
	}
	h.Sum(txid[:0])
	return txid
}

// SiacoinOutputID returns the ID of a siacoin output at the given index,
// which is calculated by hashing the concatenation of the SiacoinOutput
// Specifier, all of the fields in the transaction (except the signatures),
// and output index.
func (t Transaction) SiacoinOutputID(i uint64) SiacoinOutputID {
	return SiacoinOutputID(crypto.HashAll(
		SpecifierSiacoinOutput,
		t.SiacoinInputs,
		t.SiacoinOutputs,
		t.FileContracts,
		t.FileContractRevisions,
		t.StorageProofs,
		t.SiafundInputs,
		t.SiafundOutputs,
		t.MinerFees,
		t.ArbitraryData,
		i,
	))
}

// FileContractID returns the ID of a file contract at the given index, which
// is calculated by hashing the concatenation of the FileContract Specifier,
// all of the fields in the transaction (except the signatures), and the
// contract index.
func (t Transaction) FileContractID(i uint64) FileContractID {
	return FileContractID(crypto.HashAll(
		SpecifierFileContract,
		t.SiacoinInputs,
		t.SiacoinOutputs,
		t.FileContracts,
		t.FileContractRevisions,
		t.StorageProofs,
		t.SiafundInputs,
		t.SiafundOutputs,
		t.MinerFees,
		t.ArbitraryData,
		i,
	))
}

// SiafundOutputID returns the ID of a SiafundOutput at the given index, which
// is calculated by hashing the concatenation of the SiafundOutput Specifier,
// all of the fields in the transaction (except the signatures), and output
// index.
func (t Transaction) SiafundOutputID(i uint64) SiafundOutputID {
	return SiafundOutputID(crypto.HashAll(
		SpecifierSiafundOutput,
		t.SiacoinInputs,
		t.SiacoinOutputs,
		t.FileContracts,
		t.FileContractRevisions,
		t.StorageProofs,
		t.SiafundInputs,
		t.SiafundOutputs,
		t.MinerFees,
		t.ArbitraryData,
		i,
	))
}

// SiacoinOutputSum returns the sum of all the siacoin outputs in the
// transaction, which must match the sum of all the siacoin inputs. Siacoin
// outputs created by storage proofs and siafund outputs are not considered, as
// they were considered when the contract responsible for funding them was
// created.
func (t Transaction) SiacoinOutputSum() (sum Currency) {
	// Add the siacoin outputs.
	for _, sco := range t.SiacoinOutputs {
		sum = sum.Add(sco.Value)
	}

	// Add the file contract payouts.
	for _, fc := range t.FileContracts {
		sum = sum.Add(fc.Payout)
	}

	// Add the miner fees.
	for _, fee := range t.MinerFees {
		sum = sum.Add(fee)
	}

	return
}

// SiaClaimOutputID returns the ID of the SiacoinOutput that is created when
// the siafund output is spent. The ID is the hash the SiafundOutputID.
func (id SiafundOutputID) SiaClaimOutputID() SiacoinOutputID {
	return SiacoinOutputID(crypto.HashObject(id))
}
<commit_msg>Add marshalSiaNoSignatures method to efficiently refactor SiacoinOutputID, FileContractID, Transaction.ID().<commit_after>package types

// transaction.go defines the transaction type and all of the sub-fields of the
// transaction, as well as providing helper functions for working with
// transactions. The various IDs are designed such that, in a legal blockchain,
// it is cryptographically unlikely that any two objects would share an id.

import (
	"errors"
	"io"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
)

const (
	SpecifierLen = 16

	// UnlockHashChecksumSize is the size of the checksum used to verify
	// human-readable addresses. It is not a crypytographically secure
	// checksum, it's merely intended to prevent typos. 6 is chosen because it
	// brings the total size of the address to 38 bytes, leaving 2 bytes for
	// potential version additions in the future.
	UnlockHashChecksumSize = 6
)

// These Specifiers are used internally when calculating a type's ID. See
// Specifier for more details.
var (
	SpecifierMinerPayout          = Specifier{'m', 'i', 'n', 'e', 'r', ' ', 'p', 'a', 'y', 'o', 'u', 't'}
	SpecifierSiacoinInput         = Specifier{'s', 'i', 'a', 'c', 'o', 'i', 'n', ' ', 'i', 'n', 'p', 'u', 't'}
	SpecifierSiacoinOutput        = Specifier{'s', 'i', 'a', 'c', 'o', 'i', 'n', ' ', 'o', 'u', 't', 'p', 'u', 't'}
	SpecifierFileContract         = Specifier{'f', 'i', 'l', 'e', ' ', 'c', 'o', 'n', 't', 'r', 'a', 'c', 't'}
	SpecifierFileContractRevision = Specifier{'f', 'i', 'l', 'e', ' ', 'c', 'o', 'n', 't', 'r', 'a', 'c', 't', ' ', 'r', 'e'}
	SpecifierStorageProof         = Specifier{'s', 't', 'o', 'r', 'a', 'g', 'e', ' ', 'p', 'r', 'o', 'o', 'f'}
	SpecifierStorageProofOutput   = Specifier{'s', 't', 'o', 'r', 'a', 'g', 'e', ' ', 'p', 'r', 'o', 'o', 'f'}
	SpecifierSiafundInput         = Specifier{'s', 'i', 'a', 'f', 'u', 'n', 'd', ' ', 'i', 'n', 'p', 'u', 't'}
	SpecifierSiafundOutput        = Specifier{'s', 'i', 'a', 'f', 'u', 'n', 'd', ' ', 'o', 'u', 't', 'p', 'u', 't'}
	SpecifierClaimOutput          = Specifier{'c', 'l', 'a', 'i', 'm', ' ', 'o', 'u', 't', 'p', 'u', 't'}
	SpecifierMinerFee             = Specifier{'m', 'i', 'n', 'e', 'r', ' ', 'f', 'e', 'e'}

	ErrTransactionIDWrongLen = errors.New("input has wrong length to be an encoded transaction id")
)

type (
	// A Specifier is a fixed-length byte-array that serves two purposes. In
	// the wire protocol, they are used to identify a particular encoding
	// algorithm, signature algorithm, etc. This allows nodes to communicate on
	// their own terms; for example, to reduce bandwidth costs, a node might
	// only accept compressed messages.
	//
	// Internally, Specifiers are used to guarantee unique IDs. Various
	// consensus types have an associated ID, calculated by hashing the data
	// contained in the type. By prepending the data with Specifier, we can
	// guarantee that distinct types will never produce the same hash.
	Specifier [SpecifierLen]byte

	// IDs are used to refer to a type without revealing its contents. They
	// are constructed by hashing specific fields of the type, along with a
	// Specifier. While all of these types are hashes, defining type aliases
	// gives us type safety and makes the code more readable.
	TransactionID   crypto.Hash
	SiacoinOutputID crypto.Hash
	SiafundOutputID crypto.Hash
	FileContractID  crypto.Hash
	OutputID        crypto.Hash

	// A Transaction is an atomic component of a block. Transactions can contain
	// inputs and outputs, file contracts, storage proofs, and even arbitrary
	// data. They can also contain signatures to prove that a given party has
	// approved the transaction, or at least a particular subset of it.
	//
	// Transactions can depend on other previous transactions in the same block,
	// but transactions cannot spend outputs that they create or otherwise be
	// self-dependent.
	Transaction struct {
		SiacoinInputs         []SiacoinInput         `json:"siacoininputs"`
		SiacoinOutputs        []SiacoinOutput        `json:"siacoinoutputs"`
		FileContracts         []FileContract         `json:"filecontracts"`
		FileContractRevisions []FileContractRevision `json:"filecontractrevisions"`
		StorageProofs         []StorageProof         `json:"storageproofs"`
		SiafundInputs         []SiafundInput         `json:"siafundinputs"`
		SiafundOutputs        []SiafundOutput        `json:"siafundoutputs"`
		MinerFees             []Currency             `json:"minerfees"`
		ArbitraryData         [][]byte               `json:"arbitrarydata"`
		TransactionSignatures []TransactionSignature `json:"transactionsignatures"`
	}

	// A SiacoinInput consumes a SiacoinOutput and adds the siacoins to the set of
	// siacoins that can be spent in the transaction. The ParentID points to the
	// output that is getting consumed, and the UnlockConditions contain the rules
	// for spending the output. The UnlockConditions must match the UnlockHash of
	// the output.
	SiacoinInput struct {
		ParentID         SiacoinOutputID  `json:"parentid"`
		UnlockConditions UnlockConditions `json:"unlockconditions"`
	}

	// A SiacoinOutput holds a volume of siacoins. Outputs must be spent
	// atomically; that is, they must all be spent in the same transaction. The
	// UnlockHash is the hash of the UnlockConditions that must be fulfilled
	// in order to spend the output.
	SiacoinOutput struct {
		Value      Currency   `json:"value"`
		UnlockHash UnlockHash `json:"unlockhash"`
	}

	// A SiafundInput consumes a SiafundOutput and adds the siafunds to the set of
	// siafunds that can be spent in the transaction. The ParentID points to the
	// output that is getting consumed, and the UnlockConditions contain the rules
	// for spending the output. The UnlockConditions must match the UnlockHash of
	// the output.
	SiafundInput struct {
		ParentID         SiafundOutputID  `json:"parentid"`
		UnlockConditions UnlockConditions `json:"unlockconditions"`
		ClaimUnlockHash  UnlockHash       `json:"claimunlockhash"`
	}

	// A SiafundOutput holds a volume of siafunds. Outputs must be spent
	// atomically; that is, they must all be spent in the same transaction. The
	// UnlockHash is the hash of a set of UnlockConditions that must be fulfilled
	// in order to spend the output.
	//
	// When the SiafundOutput is spent, a SiacoinOutput is created, where:
	//
	//     SiacoinOutput.Value := (SiafundPool - ClaimStart) / 10,000
	//     SiacoinOutput.UnlockHash := SiafundOutput.ClaimUnlockHash
	//
	// When a SiafundOutput is put into a transaction, the ClaimStart must always
	// equal zero. While the transaction is being processed, the ClaimStart is set
	// to the value of the SiafundPool.
	SiafundOutput struct {
		Value      Currency   `json:"value"`
		UnlockHash UnlockHash `json:"unlockhash"`
		ClaimStart Currency   `json:"claimstart"`
	}

	// An UnlockHash is a specially constructed hash of the UnlockConditions type.
	// "Locked" values can be unlocked by providing the UnlockConditions that hash
	// to a given UnlockHash. See UnlockConditions.UnlockHash for details on how the
	// UnlockHash is constructed.
	UnlockHash crypto.Hash
)

func (t Transaction) marshalSiaNoSignatures(w io.Writer) {
	enc := encoding.NewEncoder(w)

	for i := range t.SiacoinInputs {
		t.SiacoinInputs[i].MarshalSia(w)
	}
	encoding.WriteInt(w, len((t.SiacoinOutputs)))
	for i := range t.SiacoinOutputs {
		t.SiacoinOutputs[i].MarshalSia(w)
	}
	encoding.WriteInt(w, len((t.FileContracts)))
	for i := range t.FileContracts {
		enc.Encode(t.FileContracts[i])
	}
	encoding.WriteInt(w, len((t.FileContractRevisions)))
	for i := range t.FileContractRevisions {
		enc.Encode(t.FileContractRevisions[i])
	}
	encoding.WriteInt(w, len((t.StorageProofs)))
	for i := range t.StorageProofs {
		enc.Encode(t.StorageProofs[i])
	}
	encoding.WriteInt(w, len((t.SiafundInputs)))
	for i := range t.SiafundInputs {
		enc.Encode(t.SiafundInputs[i])
	}
	encoding.WriteInt(w, len((t.SiafundOutputs)))
	for i := range t.SiafundOutputs {
		t.SiafundOutputs[i].MarshalSia(w)
	}
	encoding.WriteInt(w, len((t.MinerFees)))
	for i := range t.MinerFees {
		t.MinerFees[i].MarshalSia(w)
	}
	encoding.WriteInt(w, len((t.ArbitraryData)))
	for i := range t.ArbitraryData {
		encoding.WritePrefix(w, t.ArbitraryData[i])
	}
}

// ID returns the id of a transaction, which is taken by marshalling all of the
// fields except for the signatures and taking the hash of the result.
func (t Transaction) ID() TransactionID {
	var txid TransactionID
	h := crypto.NewHash()

	// Encode non-signature fields into hash.
	t.marshalSiaNoSignatures(h)

	// Encode signatures.
	encoding.WriteInt(h, len((t.TransactionSignatures)))
	for i := range t.TransactionSignatures {
		t.TransactionSignatures[i].MarshalSia(h)
	}

	h.Sum(txid[:0])
	return txid
}

// SiacoinOutputID returns the ID of a siacoin output at the given index,
// which is calculated by hashing the concatenation of the SiacoinOutput
// Specifier, all of the fields in the transaction (except the signatures),
// and output index.
func (t Transaction) SiacoinOutputID(i uint64) SiacoinOutputID {
	var id SiacoinOutputID
	h := crypto.NewHash()

	h.Write(SpecifierSiacoinOutput[:])
	t.marshalSiaNoSignatures(h) // Encode non-signature fields into hash.
	encoding.WriteUint64(h, i)  // Writes index of this output.

	h.Sum(id[:0])
	return id
}

// FileContractID returns the ID of a file contract at the given index, which
// is calculated by hashing the concatenation of the FileContract Specifier,
// all of the fields in the transaction (except the signatures), and the
// contract index.
func (t Transaction) FileContractID(i uint64) FileContractID {
	var id FileContractID
	h := crypto.NewHash()
	h.Write(SpecifierFileContract[:])
	t.marshalSiaNoSignatures(h) // Encode non-signature fields into hash.
	encoding.WriteUint64(h, i)  // Writes index of this output.

	h.Sum(id[:0])
	return id
}

// SiafundOutputID returns the ID of a SiafundOutput at the given index, which
// is calculated by hashing the concatenation of the SiafundOutput Specifier,
// all of the fields in the transaction (except the signatures), and output
// index.
func (t Transaction) SiafundOutputID(i uint64) SiafundOutputID {
	var id SiafundOutputID
	h := crypto.NewHash()
	h.Write(SpecifierSiafundOutput[:])
	t.marshalSiaNoSignatures(h) // Encode non-signature fields into hash.
	encoding.WriteUint64(h, i)  // Writes index of this output.

	h.Sum(id[:0])
	return id
}

// SiacoinOutputSum returns the sum of all the siacoin outputs in the
// transaction, which must match the sum of all the siacoin inputs. Siacoin
// outputs created by storage proofs and siafund outputs are not considered, as
// they were considered when the contract responsible for funding them was
// created.
func (t Transaction) SiacoinOutputSum() (sum Currency) {
	// Add the siacoin outputs.
	for _, sco := range t.SiacoinOutputs {
		sum = sum.Add(sco.Value)
	}

	// Add the file contract payouts.
	for _, fc := range t.FileContracts {
		sum = sum.Add(fc.Payout)
	}

	// Add the miner fees.
	for _, fee := range t.MinerFees {
		sum = sum.Add(fee)
	}

	return
}

// SiaClaimOutputID returns the ID of the SiacoinOutput that is created when
// the siafund output is spent. The ID is the hash the SiafundOutputID.
func (id SiafundOutputID) SiaClaimOutputID() SiacoinOutputID {
	return SiacoinOutputID(crypto.HashObject(id))
}
<|endoftext|>


<commit_before>package javascript

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/grapher2"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
	"sourcegraph.com/sourcegraph/srcgraph/scan"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	unit.Register("CommonJSPackage", &CommonJSPackage{})
	scan.Register("npm", scan.DockerScanner{defaultNPM})
	dep2.RegisterLister(&CommonJSPackage{}, defaultNPM)
	dep2.RegisterResolver(npmDependencyTargetType, dep2.DockerResolver{defaultNPM})
	grapher2.Register(&CommonJSPackage{}, grapher2.DockerGrapher{defaultNPM})
}

const nodeStdlibRepoURL = "git://github.com/joyent/node.git"

type CommonJSPackage struct {
	// If the field names of CommonJSPackage change, you need to EITHER (1)
	// update commonjs-findpkgs or (2) add a Transform func in the scanner to
	// map from the commonjs-findpkgs output to []*CommonJSPackage.

	// Dir is the directory that immediately contains the package.json
	// file (or would if one existed).
	Dir string

	// PackageJSONFile is the path to the package.json file, or empty if none
	// exists.
	PackageJSONFile string

	LibFiles  []string
	TestFiles []string
}

func (p CommonJSPackage) Name() string    { return p.Dir }
func (p CommonJSPackage) RootDir() string { return p.Dir }
func (p CommonJSPackage) sourceFiles() []string {
	return append(append([]string{}, p.LibFiles...), p.TestFiles...)
}
func (p CommonJSPackage) Paths() []string {
	f := p.sourceFiles()
	if p.PackageJSONFile != "" {
		f = append(f, p.PackageJSONFile)
	}
	return f
}

type npmVersion struct{}

var defaultNPM = &npmVersion{}

func (v *npmVersion) baseDockerfile() ([]byte, error) {
	return []byte(baseNPMDockerfile), nil
}

const baseNPMDockerfile = `FROM ubuntu:14.04
RUN apt-get update
RUN apt-get install -qy nodejs npm git`

// containerDir returns the directory in the Docker container to use for the
// local directory dir.
func containerDir(dir string) string {
	return filepath.Join("/tmp/sg", filepath.Base(dir))
}

func (v *npmVersion) BuildScanner(dir string, c *config.Repository, x *task2.Context) (*container.Command, error) {
	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}

	const (
		findpkgsNPM = "commonjs-findpkgs@0.0.2"
		findpkgsGit = "git://github.com/sourcegraph/commonjs-findpkgs.git"
		findpkgsSrc = findpkgsNPM
	)
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install -g "+findpkgsSrc+"\n")...)

	containerDir := containerDir(dir)
	cont := container.Container{
		Dockerfile: dockerfile,
		RunOptions: []string{"-v", dir + ":" + containerDir},
		Cmd:        []string{"commonjs-findpkgs"},
		Dir:        containerDir,
		Stderr:     x.Stderr,
		Stdout:     x.Stdout,
	}
	cmd := container.Command{
		Container: cont,
		Transform: func(orig []byte) ([]byte, error) {
			var pkgs []*CommonJSPackage
			err := json.Unmarshal(orig, &pkgs)
			if err != nil {
				return nil, err
			}

			return json.Marshal(pkgs)
		},
	}
	return &cmd, nil
}

func (v *npmVersion) UnmarshalSourceUnits(data []byte) ([]unit.SourceUnit, error) {
	if len(data) == 0 {
		return nil, nil
	}

	var npmPackages []*CommonJSPackage
	err := json.Unmarshal(data, &npmPackages)
	if err != nil {
		return nil, err
	}

	units := make([]unit.SourceUnit, len(npmPackages))
	for i, p := range npmPackages {
		units[i] = p
	}

	return units, nil
}

// npmDependency is a name/version pair that represents an NPM dependency. This
// pair corresponds to the object property/value pairs in package.json
// "dependency" objects.
type npmDependency struct {
	// Name is the package name of the dependency.
	Name string

	// Spec is the specifier of the version, which can be an NPM version number,
	// a tarball URL, a git/hg clone URL, etc.
	Spec string
}

const npmDependencyTargetType = "npm-dep"

func (v *npmVersion) BuildResolver(dep *dep2.RawDependency, c *config.Repository, x *task2.Context) (*container.Command, error) {
	var npmDep npmDependency
	j, _ := json.Marshal(dep.Target)
	json.Unmarshal(j, &npmDep)

	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install -g deptool@~0.0.2\n")...)

	cmd := container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			Cmd:        []string{"nodejs", "/usr/local/bin/npm-deptool", npmDep.Name + "@" + npmDep.Spec},
			Stderr:     x.Stderr,
			Stdout:     x.Stdout,
		},
		Transform: func(orig []byte) ([]byte, error) {
			// resolvedDep is output from npm-deptool.
			type npmDeptoolOutput struct {
				ResolvedURL string `json:"_resolved"`
				ID          string `json:"_id"`
				Repository  struct {
					Type string
					URL  string
				}
			}
			var resolvedDeps map[string]npmDeptoolOutput
			err := json.Unmarshal(orig, &resolvedDeps)
			if err != nil {
				return nil, err
			}

			if len(resolvedDeps) == 0 {
				return nil, fmt.Errorf("npm-deptool did not output anything for raw dependency %+v", dep)
			}
			if len(resolvedDeps) != 1 {
				return nil, fmt.Errorf("npm-deptool unexpectedly returned %d deps for raw dependency %+v", len(resolvedDeps), dep)
			}

			var resolvedDep npmDeptoolOutput
			for _, v := range resolvedDeps {
				resolvedDep = v
			}

			var toRepoCloneURL, toRevSpec string
			if strings.HasPrefix(resolvedDep.ResolvedURL, "https://registry.npmjs.org/") {
				// known npm package, so the repository refers to it
				toRepoCloneURL = resolvedDep.Repository.URL
			} else {
				// external tarball, git repo url, etc., so the repository might
				// refer to the source repo (if this is a fork) or not be
				// present at all
				u, err := url.Parse(resolvedDep.ResolvedURL)
				if err != nil {
					return nil, err
				}
				toRevSpec = u.Fragment

				u.Fragment = ""
				toRepoCloneURL = u.String()
			}

			return json.Marshal(&dep2.ResolvedTarget{
				ToRepoCloneURL:  toRepoCloneURL,
				ToUnitType:      unit.Type((&CommonJSPackage{})),
				ToUnit:          ".",
				ToVersionString: resolvedDep.ID,
				ToRevSpec:       toRevSpec,
			})
		},
	}
	return &cmd, nil
}

// List reads the "dependencies" key in the NPM package's package.json file and
// outputs the properties as raw dependencies.
func (v *npmVersion) List(dir string, unit unit.SourceUnit, c *config.Repository, x *task2.Context) ([]*dep2.RawDependency, error) {
	pkg := unit.(*CommonJSPackage)

	if pkg.PackageJSONFile == "" {
		// No package.json file, so we won't be able to find any dependencies anyway.
		return nil, nil
	}

	pkgFile := filepath.Join(dir, pkg.PackageJSONFile)

	f, err := os.Open(pkgFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var pkgjson struct {
		Dependencies    map[string]string `json:"dependencies"`
		DevDependencies map[string]string `json:"devDependencies"`
	}
	err = json.NewDecoder(f).Decode(&pkgjson)
	if err != nil {
		return nil, err
	}

	rawDeps := make([]*dep2.RawDependency, len(pkgjson.Dependencies)+len(pkgjson.DevDependencies))
	i := 0
	addDeps := func(deps map[string]string) {
		for name, spec := range deps {
			rawDeps[i] = &dep2.RawDependency{
				FromFile:   pkg.PackageJSONFile,
				TargetType: npmDependencyTargetType,
				Target:     npmDependency{Name: name, Spec: spec},
			}
			i++
		}
	}
	addDeps(pkgjson.Dependencies)
	addDeps(pkgjson.DevDependencies)

	return rawDeps, nil
}

func (v *npmVersion) BuildGrapher(dir string, u unit.SourceUnit, c *config.Repository, x *task2.Context) (*container.Command, error) {
	pkg := u.(*CommonJSPackage)

	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}

	// Install VCS tools in Docker container.
	const (
		jsgVersion = "jsg@0.0.1"
		jsgGit     = "git://github.com/sourcegraph/jsg.git"
		jsgSrc     = jsgGit
	)
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install -g "+jsgSrc+"\n")...)

	jsgCmd := []string{"nodejs", "/usr/local/bin/jsg", "--plugin", "node"}
	if len(pkg.sourceFiles()) == 0 {
		// No source files found for source unit; proceed without running grapher.
		return nil, nil
	}
	jsgCmd = append(jsgCmd, pkg.sourceFiles()...)

	// Track test files so we can set the Test field on symbols efficiently.
	isTestFile := make(map[string]struct{}, len(pkg.TestFiles))
	for _, f := range pkg.TestFiles {
		isTestFile[f] = struct{}{}
	}

	containerDir := containerDir(dir)
	cmd := container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			RunOptions: []string{"-v", dir + ":" + containerDir},
			Cmd:        jsgCmd,
			Dir:        containerDir,
			Stderr:     x.Stderr,
			Stdout:     x.Stdout,
		},
		Transform: func(in []byte) ([]byte, error) {
			var o jsgOutput
			err := json.Unmarshal(in, &o)
			if err != nil {
				return nil, err
			}

			var o2 grapher2.Output

			for _, js := range o.Symbols {
				sym, refs, propgs, docs, err := convertSymbol(js)
				if err != nil {
					return nil, err
				}

				if _, isTest := isTestFile[sym.File]; isTest {
					sym.Test = true
				}

				o2.Symbols = append(o2.Symbols, sym)
				o2.Refs = append(o2.Refs, refs...)
				// TODO(sqs): handle propgs
				_ = propgs
				o2.Docs = append(o2.Docs, docs...)
			}
			for _, jr := range o.Refs {
				ref, err := convertRef(u, jr)
				if err != nil {
					return nil, err
				}
				if ref != nil {
					o2.Refs = append(o2.Refs, ref)
				}
			}

			return json.Marshal(o2)
		},
	}

	return &cmd, nil
}

func uriOrEmpty(cloneURL string) repo.URI {
	if cloneURL == "" {
		return ""
	}
	return repo.MakeURI(cloneURL)
}
<commit_msg>bump ver<commit_after>package javascript

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/grapher2"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
	"sourcegraph.com/sourcegraph/srcgraph/scan"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	unit.Register("CommonJSPackage", &CommonJSPackage{})
	scan.Register("npm", scan.DockerScanner{defaultNPM})
	dep2.RegisterLister(&CommonJSPackage{}, defaultNPM)
	dep2.RegisterResolver(npmDependencyTargetType, dep2.DockerResolver{defaultNPM})
	grapher2.Register(&CommonJSPackage{}, grapher2.DockerGrapher{defaultNPM})
}

const nodeStdlibRepoURL = "git://github.com/joyent/node.git"

type CommonJSPackage struct {
	// If the field names of CommonJSPackage change, you need to EITHER (1)
	// update commonjs-findpkgs or (2) add a Transform func in the scanner to
	// map from the commonjs-findpkgs output to []*CommonJSPackage.

	// Dir is the directory that immediately contains the package.json
	// file (or would if one existed).
	Dir string

	// PackageJSONFile is the path to the package.json file, or empty if none
	// exists.
	PackageJSONFile string

	LibFiles  []string
	TestFiles []string
}

func (p CommonJSPackage) Name() string    { return p.Dir }
func (p CommonJSPackage) RootDir() string { return p.Dir }
func (p CommonJSPackage) sourceFiles() []string {
	return append(append([]string{}, p.LibFiles...), p.TestFiles...)
}
func (p CommonJSPackage) Paths() []string {
	f := p.sourceFiles()
	if p.PackageJSONFile != "" {
		f = append(f, p.PackageJSONFile)
	}
	return f
}

type npmVersion struct{}

var defaultNPM = &npmVersion{}

func (v *npmVersion) baseDockerfile() ([]byte, error) {
	return []byte(baseNPMDockerfile), nil
}

const baseNPMDockerfile = `FROM ubuntu:14.04
RUN apt-get update
RUN apt-get install -qy nodejs npm git`

// containerDir returns the directory in the Docker container to use for the
// local directory dir.
func containerDir(dir string) string {
	return filepath.Join("/tmp/sg", filepath.Base(dir))
}

func (v *npmVersion) BuildScanner(dir string, c *config.Repository, x *task2.Context) (*container.Command, error) {
	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}

	const (
		findpkgsNPM = "commonjs-findpkgs@0.0.3"
		findpkgsGit = "git://github.com/sourcegraph/commonjs-findpkgs.git"
		findpkgsSrc = findpkgsNPM
	)
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install -g "+findpkgsSrc+"\n")...)

	containerDir := containerDir(dir)
	cont := container.Container{
		Dockerfile: dockerfile,
		RunOptions: []string{"-v", dir + ":" + containerDir},
		Cmd:        []string{"commonjs-findpkgs"},
		Dir:        containerDir,
		Stderr:     x.Stderr,
		Stdout:     x.Stdout,
	}
	cmd := container.Command{
		Container: cont,
		Transform: func(orig []byte) ([]byte, error) {
			var pkgs []*CommonJSPackage
			err := json.Unmarshal(orig, &pkgs)
			if err != nil {
				return nil, err
			}

			return json.Marshal(pkgs)
		},
	}
	return &cmd, nil
}

func (v *npmVersion) UnmarshalSourceUnits(data []byte) ([]unit.SourceUnit, error) {
	if len(data) == 0 {
		return nil, nil
	}

	var npmPackages []*CommonJSPackage
	err := json.Unmarshal(data, &npmPackages)
	if err != nil {
		return nil, err
	}

	units := make([]unit.SourceUnit, len(npmPackages))
	for i, p := range npmPackages {
		units[i] = p
	}

	return units, nil
}

// npmDependency is a name/version pair that represents an NPM dependency. This
// pair corresponds to the object property/value pairs in package.json
// "dependency" objects.
type npmDependency struct {
	// Name is the package name of the dependency.
	Name string

	// Spec is the specifier of the version, which can be an NPM version number,
	// a tarball URL, a git/hg clone URL, etc.
	Spec string
}

const npmDependencyTargetType = "npm-dep"

func (v *npmVersion) BuildResolver(dep *dep2.RawDependency, c *config.Repository, x *task2.Context) (*container.Command, error) {
	var npmDep npmDependency
	j, _ := json.Marshal(dep.Target)
	json.Unmarshal(j, &npmDep)

	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install -g deptool@~0.0.2\n")...)

	cmd := container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			Cmd:        []string{"nodejs", "/usr/local/bin/npm-deptool", npmDep.Name + "@" + npmDep.Spec},
			Stderr:     x.Stderr,
			Stdout:     x.Stdout,
		},
		Transform: func(orig []byte) ([]byte, error) {
			// resolvedDep is output from npm-deptool.
			type npmDeptoolOutput struct {
				ResolvedURL string `json:"_resolved"`
				ID          string `json:"_id"`
				Repository  struct {
					Type string
					URL  string
				}
			}
			var resolvedDeps map[string]npmDeptoolOutput
			err := json.Unmarshal(orig, &resolvedDeps)
			if err != nil {
				return nil, err
			}

			if len(resolvedDeps) == 0 {
				return nil, fmt.Errorf("npm-deptool did not output anything for raw dependency %+v", dep)
			}
			if len(resolvedDeps) != 1 {
				return nil, fmt.Errorf("npm-deptool unexpectedly returned %d deps for raw dependency %+v", len(resolvedDeps), dep)
			}

			var resolvedDep npmDeptoolOutput
			for _, v := range resolvedDeps {
				resolvedDep = v
			}

			var toRepoCloneURL, toRevSpec string
			if strings.HasPrefix(resolvedDep.ResolvedURL, "https://registry.npmjs.org/") {
				// known npm package, so the repository refers to it
				toRepoCloneURL = resolvedDep.Repository.URL
			} else {
				// external tarball, git repo url, etc., so the repository might
				// refer to the source repo (if this is a fork) or not be
				// present at all
				u, err := url.Parse(resolvedDep.ResolvedURL)
				if err != nil {
					return nil, err
				}
				toRevSpec = u.Fragment

				u.Fragment = ""
				toRepoCloneURL = u.String()
			}

			return json.Marshal(&dep2.ResolvedTarget{
				ToRepoCloneURL:  toRepoCloneURL,
				ToUnitType:      unit.Type((&CommonJSPackage{})),
				ToUnit:          ".",
				ToVersionString: resolvedDep.ID,
				ToRevSpec:       toRevSpec,
			})
		},
	}
	return &cmd, nil
}

// List reads the "dependencies" key in the NPM package's package.json file and
// outputs the properties as raw dependencies.
func (v *npmVersion) List(dir string, unit unit.SourceUnit, c *config.Repository, x *task2.Context) ([]*dep2.RawDependency, error) {
	pkg := unit.(*CommonJSPackage)

	if pkg.PackageJSONFile == "" {
		// No package.json file, so we won't be able to find any dependencies anyway.
		return nil, nil
	}

	pkgFile := filepath.Join(dir, pkg.PackageJSONFile)

	f, err := os.Open(pkgFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var pkgjson struct {
		Dependencies    map[string]string `json:"dependencies"`
		DevDependencies map[string]string `json:"devDependencies"`
	}
	err = json.NewDecoder(f).Decode(&pkgjson)
	if err != nil {
		return nil, err
	}

	rawDeps := make([]*dep2.RawDependency, len(pkgjson.Dependencies)+len(pkgjson.DevDependencies))
	i := 0
	addDeps := func(deps map[string]string) {
		for name, spec := range deps {
			rawDeps[i] = &dep2.RawDependency{
				FromFile:   pkg.PackageJSONFile,
				TargetType: npmDependencyTargetType,
				Target:     npmDependency{Name: name, Spec: spec},
			}
			i++
		}
	}
	addDeps(pkgjson.Dependencies)
	addDeps(pkgjson.DevDependencies)

	return rawDeps, nil
}

func (v *npmVersion) BuildGrapher(dir string, u unit.SourceUnit, c *config.Repository, x *task2.Context) (*container.Command, error) {
	pkg := u.(*CommonJSPackage)

	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}

	// Install VCS tools in Docker container.
	const (
		jsgVersion = "jsg@0.0.1"
		jsgGit     = "git://github.com/sourcegraph/jsg.git"
		jsgSrc     = jsgGit
	)
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install -g "+jsgSrc+"\n")...)

	jsgCmd := []string{"nodejs", "/usr/local/bin/jsg", "--plugin", "node"}
	if len(pkg.sourceFiles()) == 0 {
		// No source files found for source unit; proceed without running grapher.
		return nil, nil
	}
	jsgCmd = append(jsgCmd, pkg.sourceFiles()...)

	// Track test files so we can set the Test field on symbols efficiently.
	isTestFile := make(map[string]struct{}, len(pkg.TestFiles))
	for _, f := range pkg.TestFiles {
		isTestFile[f] = struct{}{}
	}

	containerDir := containerDir(dir)
	cmd := container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			RunOptions: []string{"-v", dir + ":" + containerDir},
			Cmd:        jsgCmd,
			Dir:        containerDir,
			Stderr:     x.Stderr,
			Stdout:     x.Stdout,
		},
		Transform: func(in []byte) ([]byte, error) {
			var o jsgOutput
			err := json.Unmarshal(in, &o)
			if err != nil {
				return nil, err
			}

			var o2 grapher2.Output

			for _, js := range o.Symbols {
				sym, refs, propgs, docs, err := convertSymbol(js)
				if err != nil {
					return nil, err
				}

				if _, isTest := isTestFile[sym.File]; isTest {
					sym.Test = true
				}

				o2.Symbols = append(o2.Symbols, sym)
				o2.Refs = append(o2.Refs, refs...)
				// TODO(sqs): handle propgs
				_ = propgs
				o2.Docs = append(o2.Docs, docs...)
			}
			for _, jr := range o.Refs {
				ref, err := convertRef(u, jr)
				if err != nil {
					return nil, err
				}
				if ref != nil {
					o2.Refs = append(o2.Refs, ref)
				}
			}

			return json.Marshal(o2)
		},
	}

	return &cmd, nil
}

func uriOrEmpty(cloneURL string) repo.URI {
	if cloneURL == "" {
		return ""
	}
	return repo.MakeURI(cloneURL)
}
<|endoftext|>


<commit_before>// Package zipexe attempts to open an executable binary file as a zip file.
package zipexe

import (
	"archive/zip"
	"debug/elf"
	"debug/macho"
	"debug/pe"
	"errors"
	"io"
	"os"
)

// Opens a zip file by path.
func Open(path string) (*zip.Reader, error) {
	_, rd, err := OpenCloser(path)
	return rd, err
}

// OpenCloser is like Open but returns an additional Closer to avoid leaking open files.
func OpenCloser(path string) (io.Closer, *zip.Reader, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, nil, err
	}
	finfo, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}
	zr, err := NewReader(file, finfo.Size())
	if err != nil {
		return nil, nil, err
	}
	return file, zr, nil
}

// Open a zip file, specially handling various binaries that may have been
// augmented with zip data.
func NewReader(rda io.ReaderAt, size int64) (*zip.Reader, error) {
	handlers := []func(io.ReaderAt, int64) (*zip.Reader, error){
		zip.NewReader,
		zipExeReaderMacho,
		zipExeReaderElf,
		zipExeReaderPe,
	}

	for _, handler := range handlers {
		zfile, err := handler(rda, size)
		if err == nil {
			return zfile, nil
		}
	}
	return nil, errors.New("Couldn't Open As Executable")
}

// zipExeReaderMacho treats the file as a Mach-O binary
// (Mac OS X / Darwin executable) and attempts to find a zip archive.
func zipExeReaderMacho(rda io.ReaderAt, size int64) (*zip.Reader, error) {
	file, err := macho.NewFile(rda)
	if err != nil {
		return nil, err
	}

	var max int64
	for _, load := range file.Loads {
		seg, ok := load.(*macho.Segment)
		if ok {
			// Check if the segment contains a zip file
			if zfile, err := zip.NewReader(seg, int64(seg.Filesz)); err == nil {
				return zfile, nil
			}

			// Otherwise move end of file pointer
			end := int64(seg.Offset + seg.Filesz)
			if end > max {
				max = end
			}
		}
	}

	// No zip file within binary, try appended to end
	section := io.NewSectionReader(rda, max, size-max)
	return zip.NewReader(section, section.Size())
}

// zipExeReaderPe treats the file as a Portable Exectuable binary
// (Windows executable) and attempts to find a zip archive.
func zipExeReaderPe(rda io.ReaderAt, size int64) (*zip.Reader, error) {
	file, err := pe.NewFile(rda)
	if err != nil {
		return nil, err
	}

	var max int64
	for _, sec := range file.Sections {
		// Check if this section has a zip file
		if zfile, err := zip.NewReader(sec, int64(sec.Size)); err == nil {
			return zfile, nil
		}

		// Otherwise move end of file pointer
		end := int64(sec.Offset + sec.Size)
		if end > max {
			max = end
		}
	}

	// No zip file within binary, try appended to end
	section := io.NewSectionReader(rda, max, size-max)
	return zip.NewReader(section, section.Size())
}

// zipExeReaderElf treats the file as a ELF binary
// (linux/BSD/etc... executable) and attempts to find a zip archive.
func zipExeReaderElf(rda io.ReaderAt, size int64) (*zip.Reader, error) {
	file, err := elf.NewFile(rda)
	if err != nil {
		return nil, err
	}

	var max int64
	for _, sect := range file.Sections {
		if sect.Type == elf.SHT_NOBITS {
			continue
		}

		// Check if this section has a zip file
		if zfile, err := zip.NewReader(sect, int64(sect.Size)); err == nil {
			return zfile, nil
		}

		// Otherwise move end of file pointer
		end := int64(sect.Offset + sect.Size)
		if end > max {
			max = end
		}
	}

	// No zip file within binary, try appended to end
	section := io.NewSectionReader(rda, max, size-max)
	return zip.NewReader(section, section.Size())
}
<commit_msg>ignore elf sections without ReaderAt<commit_after>// Package zipexe attempts to open an executable binary file as a zip file.
package zipexe

import (
	"archive/zip"
	"debug/elf"
	"debug/macho"
	"debug/pe"
	"errors"
	"io"
	"os"
)

// Opens a zip file by path.
func Open(path string) (*zip.Reader, error) {
	_, rd, err := OpenCloser(path)
	return rd, err
}

// OpenCloser is like Open but returns an additional Closer to avoid leaking open files.
func OpenCloser(path string) (io.Closer, *zip.Reader, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, nil, err
	}
	finfo, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}
	zr, err := NewReader(file, finfo.Size())
	if err != nil {
		return nil, nil, err
	}
	return file, zr, nil
}

// Open a zip file, specially handling various binaries that may have been
// augmented with zip data.
func NewReader(rda io.ReaderAt, size int64) (*zip.Reader, error) {
	handlers := []func(io.ReaderAt, int64) (*zip.Reader, error){
		zip.NewReader,
		zipExeReaderMacho,
		zipExeReaderElf,
		zipExeReaderPe,
	}

	for _, handler := range handlers {
		zfile, err := handler(rda, size)
		if err == nil {
			return zfile, nil
		}
	}
	return nil, errors.New("Couldn't Open As Executable")
}

// zipExeReaderMacho treats the file as a Mach-O binary
// (Mac OS X / Darwin executable) and attempts to find a zip archive.
func zipExeReaderMacho(rda io.ReaderAt, size int64) (*zip.Reader, error) {
	file, err := macho.NewFile(rda)
	if err != nil {
		return nil, err
	}

	var max int64
	for _, load := range file.Loads {
		seg, ok := load.(*macho.Segment)
		if ok {
			// Check if the segment contains a zip file
			if zfile, err := zip.NewReader(seg, int64(seg.Filesz)); err == nil {
				return zfile, nil
			}

			// Otherwise move end of file pointer
			end := int64(seg.Offset + seg.Filesz)
			if end > max {
				max = end
			}
		}
	}

	// No zip file within binary, try appended to end
	section := io.NewSectionReader(rda, max, size-max)
	return zip.NewReader(section, section.Size())
}

// zipExeReaderPe treats the file as a Portable Exectuable binary
// (Windows executable) and attempts to find a zip archive.
func zipExeReaderPe(rda io.ReaderAt, size int64) (*zip.Reader, error) {
	file, err := pe.NewFile(rda)
	if err != nil {
		return nil, err
	}

	var max int64
	for _, sec := range file.Sections {
		// Check if this section has a zip file
		if zfile, err := zip.NewReader(sec, int64(sec.Size)); err == nil {
			return zfile, nil
		}

		// Otherwise move end of file pointer
		end := int64(sec.Offset + sec.Size)
		if end > max {
			max = end
		}
	}

	// No zip file within binary, try appended to end
	section := io.NewSectionReader(rda, max, size-max)
	return zip.NewReader(section, section.Size())
}

// zipExeReaderElf treats the file as a ELF binary
// (linux/BSD/etc... executable) and attempts to find a zip archive.
func zipExeReaderElf(rda io.ReaderAt, size int64) (*zip.Reader, error) {
	file, err := elf.NewFile(rda)
	if err != nil {
		return nil, err
	}

	var max int64
	for _, sect := range file.Sections {
		if sect.Type == elf.SHT_NOBITS {
			continue
		}

		// Check if this section has a zip file
		if sect.ReaderAt != nil {
			if zfile, err := zip.NewReader(sect, int64(sect.Size)); err == nil {
				return zfile, nil
			}
		}

		// Otherwise move end of file pointer
		end := int64(sect.Offset + sect.Size)
		if end > max {
			max = end
		}
	}

	// No zip file within binary, try appended to end
	section := io.NewSectionReader(rda, max, size-max)
	return zip.NewReader(section, section.Size())
}
<|endoftext|>


<commit_before>package cmgo

import (
    "fmt"
    "gopkg.in/mgo.v2"
)

type MGOClient struct {
    session *mgo.Session
    database *mgo.Database
    collection *mgo.Collection
}

func New(host, port, username, password, dbname string) (*MGOClient, error) {
    mgoClient := &MGOClient{
        session: nil,
        database: nil,
        collection: nil,
    }

    mongoAddr := fmt.Sprintf("mongodb://%v:%v@%v:%v/%v", username, password, host, port, dbname)

    // 连接数据库
    session, err := mgo.Dial(mongoAddr)
    if err != nil {
        return nil, err
    }

    mgoClient.session = session
    mgoClient.database = session.DB(dbname)

    return mgoClient, nil
}

func (this *MGOClient) Close() {
    this.session.Close()
}

func (this *MGOClient) UseCollection(collection_name string) {
    this.collection = this.database.C(collection_name)
}

func (this *MGOClient) Insert(docs ...interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    return this.collection.Insert(docs)
}

func (this *MGOClient) Remove(selector interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    return this.collection.Remove(selector)
}

func (this *MGOClient) RemoveAll(selector interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    _, err := this.collection.RemoveAll(selector)
    return err
}

func (this *MGOClient) Update(selector interface{}, update interface{}) error {
    if this.collection == nil {
       return  fmt.Errorf("invalid collections object")
    }

    return this.collection.Update(selector, update)
}

func (this *MGOClient) UpdateAll(selector interface{}, update interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    _, err := this.collection.UpdateAll(selector, update)

    return err
}

func (this *MGOClient) Upsert(selector interface{}, update interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    _, err := this.collection.Upsert(selector, update)

    return err
}

func (this *MGOClient) Find(query interface{}) *mgo.Query {
    if this.collection == nil {
        return nil
    }

    return this.collection.Find(query)
}

func (this *MGOClient) DropCollection() error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    return this.collection.DropCollection()
}

func (this *MGOClient) Count() (n int, err error) {
    if this.collection == nil {
        return 0, fmt.Errorf("invalid collections object")
    }

    return this.collection.Count()
}

<commit_msg>impove the cmgo api<commit_after>package cmgo

import (
    "fmt"
    "gopkg.in/mgo.v2"
)

type MGOClient struct {
    session *mgo.Session
    database *mgo.Database
    collection *mgo.Collection
}

func New(host, port, username, password, dbname string) (*MGOClient, error) {
    mgoClient := &MGOClient{
        session: nil,
        database: nil,
        collection: nil,
    }

    if host == "" {
        return nil, fmt.Errorf("host addr not set")
    }

    var mongoAddr string
    if username != "" && password != "" && port != "" && dbname != "" {
        mongoAddr = fmt.Sprintf("mongodb://%v:%v@%v:%v/%v", username, password, host, port, dbname)
    } else if username != "" && password != "" && port != "" {
        mongoAddr = fmt.Sprintf("mongodb://%v:%v@%v:%v", username, password, host, port)
    } else if username != "" && password != "" {
        mongoAddr = fmt.Sprintf("mongodb://%v:%v@%v", username, password, host)
    } else if port != "" && dbname != "" {
        mongoAddr = fmt.Sprintf("mongodb://%v:%v/%v", host, port, dbname)
    } else if port != "" {
        mongoAddr = fmt.Sprintf("mongodb://v:%v", host, port)
    } else {
        mongoAddr = fmt.Sprintf("mongodb://v", host)
    }

    // 连接数据库
    session, err := mgo.Dial(mongoAddr)
    if err != nil {
        return nil, err
    }

    mgoClient.session = session

    if dbname != "" {
        mgoClient.database = session.DB(dbname)
    }

    return mgoClient, nil
}

func (this *MGOClient) Close() {
    this.session.Close()
}

//if the dbname is empty, the "test" database is selected.
func (this *MGOClient) UseDB(dbname string) error {
    if this.session == nil {
        return fmt.Errorf("invalid session object")
    }

    this.database = this.session.DB(dbname)
    return nil
}

func (this *MGOClient) UseCollection(collection_name string) error {
    if this.database == nil {
        fmt.Errorf("invalid database object")
    }

    this.collection = this.database.C(collection_name)
    return nil
}

func (this *MGOClient) GetCurrentCollection() *mgo.Collection {
    return this.collection
}

func (this *MGOClient) Insert(doc interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    return this.collection.Insert(doc)
}

func (this *MGOClient) Remove(selector interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    return this.collection.Remove(selector)
}

func (this *MGOClient) RemoveAll(selector interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    _, err := this.collection.RemoveAll(selector)
    return err
}

func (this *MGOClient) Update(selector interface{}, update interface{}) error {
    if this.collection == nil {
       return  fmt.Errorf("invalid collections object")
    }

    return this.collection.Update(selector, update)
}

func (this *MGOClient) UpdateAll(selector interface{}, update interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    _, err := this.collection.UpdateAll(selector, update)

    return err
}

func (this *MGOClient) Upsert(selector interface{}, update interface{}) error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    _, err := this.collection.Upsert(selector, update)

    return err
}

func (this *MGOClient) Find(query interface{}) *mgo.Query {
    if this.collection == nil {
        return nil
    }

    return this.collection.Find(query)
}

func (this *MGOClient) DropCollection() error {
    if this.collection == nil {
        return fmt.Errorf("invalid collections object")
    }

    return this.collection.DropCollection()
}

func (this *MGOClient) Count() (n int, err error) {
    if this.collection == nil {
        return 0, fmt.Errorf("invalid collections object")
    }

    return this.collection.Count()
}

<|endoftext|>


<commit_before>package core

import (
	"context"
	"errors"
	"math"
	"strings"
	"sync"

	"github.com/golang/glog"

	"github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/lpms/ffmpeg"
)

var ErrTranscoderBusy = errors.New("TranscoderBusy")
var ErrTranscoderStopped = errors.New("TranscoderStopped")

type TranscoderSession interface {
	Transcoder
	Stop()
}

type newTranscoderFn func(device string) TranscoderSession

type LoadBalancingTranscoder struct {
	transcoders []string // Slice of device IDs
	newT        newTranscoderFn

	// The following fields need to be protected by the mutex `mu`
	mu       *sync.RWMutex
	load     map[string]int
	sessions map[string]*transcoderSession
	idx      int // Ensures a non-tapered work distribution
}

func NewLoadBalancingTranscoder(devices string, newTranscoderFn newTranscoderFn) Transcoder {
	d := strings.Split(devices, ",")
	return &LoadBalancingTranscoder{
		transcoders: d,
		newT:        newTranscoderFn,
		mu:          &sync.RWMutex{},
		load:        make(map[string]int),
		sessions:    make(map[string]*transcoderSession),
	}
}

func (lb *LoadBalancingTranscoder) Transcode(md *SegTranscodingMetadata) (*TranscodeData, error) {

	lb.mu.RLock()
	session, exists := lb.sessions[string(md.ManifestID)]
	lb.mu.RUnlock()
	if exists {
		glog.V(common.DEBUG).Info("LB: Using existing transcode session for ", session.key)
	} else {
		var err error
		session, err = lb.createSession(md)
		if err != nil {
			return nil, err
		}
	}
	return session.Transcode(md)
}

func (lb *LoadBalancingTranscoder) createSession(md *SegTranscodingMetadata) (*transcoderSession, error) {

	lb.mu.Lock()
	defer lb.mu.Unlock()

	job := string(md.ManifestID)
	if session, exists := lb.sessions[job]; exists {
		glog.V(common.DEBUG).Info("Attempted to create session but already exists ", session.key)
		return session, nil
	}

	glog.V(common.DEBUG).Info("LB: Creating transcode session for ", job)
	transcoder := lb.leastLoaded()

	// Acquire transcode session. Map to job id + assigned transcoder
	key := job + "_" + transcoder
	costEstimate := calculateCost(md.Profiles)
	session := &transcoderSession{
		transcoder:  lb.newT(transcoder),
		key:         key,
		sender:      make(chan *transcoderParams, maxSegmentChannels),
		makeContext: transcodeLoopContext,
	}
	lb.sessions[job] = session
	lb.load[transcoder] += costEstimate
	lb.idx = (lb.idx + 1) % len(lb.transcoders)

	// Local cleanup function
	cleanupSession := func() {
		lb.mu.Lock()
		defer lb.mu.Unlock()
		_, exists := lb.sessions[job]
		if !exists {
			return
		}
		delete(lb.sessions, job)
		lb.load[transcoder] -= costEstimate
		glog.V(common.DEBUG).Info("LB: Deleted transcode session for ", session.key)
	}

	go func() {
		session.loop()
		cleanupSession()
	}()

	glog.V(common.DEBUG).Info("LB: Created transcode session for ", session.key)
	return session, nil
}

// Find the lowest loaded transcoder.
// Expects the mutex `lb.mu` to be locked by the caller.
func (lb *LoadBalancingTranscoder) leastLoaded() string {
	min, idx := math.MaxInt64, 0
	for i := 0; i < len(lb.transcoders); i++ {
		k := (i + lb.idx) % len(lb.transcoders)
		if lb.load[lb.transcoders[k]] < min {
			min = lb.load[lb.transcoders[k]]
			idx = k
		}
	}
	return lb.transcoders[idx]
}

type transcoderParams struct {
	md  *SegTranscodingMetadata
	res chan struct {
		*TranscodeData
		error
	}
}

type transcoderSession struct {
	transcoder TranscoderSession
	key        string

	sender      chan *transcoderParams
	makeContext func() (context.Context, context.CancelFunc)
}

func (sess *transcoderSession) loop() {
	defer func() {
		sess.transcoder.Stop()
		// Attempt to drain any pending messages in the channel.
		// Otherwise, write a message into the channel to fill it up.
		// Since we know the channel is buffered with size 1, any
		// successful writes here mean the channel is full
		// and we can safely exit immediately knowing that subsequent writes
		// will be rejected
		for {
			select {
			case params := <-sess.sender:
				params.res <- struct {
					*TranscodeData
					error
				}{nil, ErrTranscoderStopped}
			case sess.sender <- &transcoderParams{}:
				return
			default:
				continue
			}
		}
	}()

	// Run everything on a single loop to mitigate threading issues,
	//   especially around transcoder cleanup
	for {
		ctx, cancel := sess.makeContext()
		select {
		case <-ctx.Done():
			// Terminate the session after a period of inactivity
			glog.V(common.DEBUG).Info("LB: Transcode loop timed out for ", sess.key)
			return
		case params := <-sess.sender:
			cancel()
			res, err :=
				sess.transcoder.Transcode(params.md)
			params.res <- struct {
				*TranscodeData
				error
			}{res, err}
			if err != nil {
				glog.V(common.DEBUG).Info("LB: Stopping transcoder due to error for ", sess.key)
				return
			}
		}
	}
}

func (sess *transcoderSession) Transcode(md *SegTranscodingMetadata) (*TranscodeData, error) {
	params := &transcoderParams{md: md,
		res: make(chan struct {
			*TranscodeData
			error
		})}
	select {
	case sess.sender <- params:
		glog.V(common.DEBUG).Info("LB: Transcode submitted for ", sess.key)
	default:
		glog.V(common.DEBUG).Info("LB: Transcoder was busy; exiting ", sess.key)
		return nil, ErrTranscoderBusy
	}
	res := <-params.res
	return res.TranscodeData, res.error
}

func calculateCost(profiles []ffmpeg.VideoProfile) int {
	cost := 0
	for _, v := range profiles {
		w, h, err := ffmpeg.VideoProfileResolution(v)
		if err != nil {
			continue
		}
		framerate := int(v.Framerate)
		if 0 == framerate {
			// passthrough; estimate 30fps for load balancing purposes
			framerate = 30
		}
		framerateDen := int(v.FramerateDen)
		if 0 == framerateDen {
			// denominator unset; treat as 1
			framerateDen = 1
		}
		cost += w * h * (framerate / framerateDen) // TODO incorporate duration
	}
	return cost
}
<commit_msg>core: fix lb session cleanup race<commit_after>package core

import (
	"context"
	"errors"
	"math"
	"strings"
	"sync"

	"github.com/golang/glog"

	"github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/lpms/ffmpeg"
)

var ErrTranscoderBusy = errors.New("TranscoderBusy")
var ErrTranscoderStopped = errors.New("TranscoderStopped")

type TranscoderSession interface {
	Transcoder
	Stop()
}

type newTranscoderFn func(device string) TranscoderSession

type LoadBalancingTranscoder struct {
	transcoders []string // Slice of device IDs
	newT        newTranscoderFn

	// The following fields need to be protected by the mutex `mu`
	mu       *sync.RWMutex
	load     map[string]int
	sessions map[string]*transcoderSession
	idx      int // Ensures a non-tapered work distribution
}

func NewLoadBalancingTranscoder(devices string, newTranscoderFn newTranscoderFn) Transcoder {
	d := strings.Split(devices, ",")
	return &LoadBalancingTranscoder{
		transcoders: d,
		newT:        newTranscoderFn,
		mu:          &sync.RWMutex{},
		load:        make(map[string]int),
		sessions:    make(map[string]*transcoderSession),
	}
}

func (lb *LoadBalancingTranscoder) Transcode(md *SegTranscodingMetadata) (*TranscodeData, error) {

	lb.mu.RLock()
	session, exists := lb.sessions[string(md.ManifestID)]
	lb.mu.RUnlock()
	if exists {
		glog.V(common.DEBUG).Info("LB: Using existing transcode session for ", session.key)
	} else {
		var err error
		session, err = lb.createSession(md)
		if err != nil {
			return nil, err
		}
	}
	return session.Transcode(md)
}

func (lb *LoadBalancingTranscoder) createSession(md *SegTranscodingMetadata) (*transcoderSession, error) {

	lb.mu.Lock()
	defer lb.mu.Unlock()

	job := string(md.ManifestID)
	if session, exists := lb.sessions[job]; exists {
		glog.V(common.DEBUG).Info("Attempted to create session but already exists ", session.key)
		return session, nil
	}

	glog.V(common.DEBUG).Info("LB: Creating transcode session for ", job)
	transcoder := lb.leastLoaded()

	// Acquire transcode session. Map to job id + assigned transcoder
	key := job + "_" + transcoder
	costEstimate := calculateCost(md.Profiles)
	session := &transcoderSession{
		transcoder:  lb.newT(transcoder),
		key:         key,
		done:        make(chan struct{}),
		sender:      make(chan *transcoderParams, maxSegmentChannels),
		makeContext: transcodeLoopContext,
	}
	lb.sessions[job] = session
	lb.load[transcoder] += costEstimate
	lb.idx = (lb.idx + 1) % len(lb.transcoders)

	// Local cleanup function
	cleanupSession := func() {
		lb.mu.Lock()
		defer lb.mu.Unlock()
		_, exists := lb.sessions[job]
		if !exists {
			return
		}
		delete(lb.sessions, job)
		lb.load[transcoder] -= costEstimate
		glog.V(common.DEBUG).Info("LB: Deleted transcode session for ", session.key)
	}

	go func() {
		session.loop()
		cleanupSession()
	}()

	glog.V(common.DEBUG).Info("LB: Created transcode session for ", session.key)
	return session, nil
}

// Find the lowest loaded transcoder.
// Expects the mutex `lb.mu` to be locked by the caller.
func (lb *LoadBalancingTranscoder) leastLoaded() string {
	min, idx := math.MaxInt64, 0
	for i := 0; i < len(lb.transcoders); i++ {
		k := (i + lb.idx) % len(lb.transcoders)
		if lb.load[lb.transcoders[k]] < min {
			min = lb.load[lb.transcoders[k]]
			idx = k
		}
	}
	return lb.transcoders[idx]
}

type transcoderParams struct {
	md  *SegTranscodingMetadata
	res chan struct {
		*TranscodeData
		error
	}
}

type transcoderSession struct {
	transcoder TranscoderSession
	key        string

	sender      chan *transcoderParams
	done        chan struct{}
	makeContext func() (context.Context, context.CancelFunc)
}

func (sess *transcoderSession) loop() {
	defer func() {
		sess.transcoder.Stop()
		// Close the done channel to signal the sender(s) that the
		// transcode loop has stopped
		close(sess.done)
	}()

	// Run everything on a single loop to mitigate threading issues,
	//   especially around transcoder cleanup
	for {
		ctx, cancel := sess.makeContext()
		select {
		case <-ctx.Done():
			// Terminate the session after a period of inactivity
			glog.V(common.DEBUG).Info("LB: Transcode loop timed out for ", sess.key)
			return
		case params := <-sess.sender:
			cancel()
			res, err :=
				sess.transcoder.Transcode(params.md)
			params.res <- struct {
				*TranscodeData
				error
			}{res, err}
			if err != nil {
				glog.V(common.DEBUG).Info("LB: Stopping transcoder due to error for ", sess.key)
				return
			}
		}
	}
}

func (sess *transcoderSession) Transcode(md *SegTranscodingMetadata) (*TranscodeData, error) {
	params := &transcoderParams{md: md,
		res: make(chan struct {
			*TranscodeData
			error
		})}
	select {
	case sess.sender <- params:
		glog.V(common.DEBUG).Info("LB: Transcode submitted for ", sess.key)
	default:
		glog.V(common.DEBUG).Info("LB: Transcoder was busy; exiting ", sess.key)
		return nil, ErrTranscoderBusy
	}
	select {
	case res := <-params.res:
		return res.TranscodeData, res.error
	case <-sess.done:
		return nil, ErrTranscoderStopped
	}
}

func calculateCost(profiles []ffmpeg.VideoProfile) int {
	cost := 0
	for _, v := range profiles {
		w, h, err := ffmpeg.VideoProfileResolution(v)
		if err != nil {
			continue
		}
		framerate := int(v.Framerate)
		if 0 == framerate {
			// passthrough; estimate 30fps for load balancing purposes
			framerate = 30
		}
		framerateDen := int(v.FramerateDen)
		if 0 == framerateDen {
			// denominator unset; treat as 1
			framerateDen = 1
		}
		cost += w * h * (framerate / framerateDen) // TODO incorporate duration
	}
	return cost
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package refactor

import (
	"fmt"

	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/readwriter"
	"github.com/brocaar/lorawan"
)

type RPacket interface {
	Packet
	Metadata() Metadata
	Payload() lorawan.PHYPayload
	DevEUI() lorawan.EUI64
}

// rpacket implements the core.RPacket interface
type rpacket struct {
	metadata Metadata
	payload  lorawan.PHYPayload
}

// NewRPacket construct a new router packet given a payload and metadata
func NewRPacket(payload lorawan.PHYPayload, metadata Metadata) (RPacket, error) {
	packet := rpacket{payload: payload, metadata: metadata}

	// Check and extract the devEUI
	if payload.MACPayload == nil {
		return nil, errors.New(errors.Structural, "MACPAyload should not be empty")
	}

	_, ok := payload.MACPayload.(*lorawan.MACPayload)
	if !ok {
		return nil, errors.New(errors.Structural, "Packet does not carry a MACPayload")
	}

	return &packet, nil
}

// DevEUI implements the core.BPacket interface
func (p rpacket) DevEUI() lorawan.EUI64 {
	var devEUI lorawan.EUI64
	copy(devEUI[4:], p.payload.MACPayload.(*lorawan.MACPayload).FHDR.DevAddr[:])
	return devEUI
}

// Metadata implements the core.RPacket interface
func (p rpacket) Metadata() Metadata {
	return p.metadata
}

// Payload implements the core.RPacket interface
func (p rpacket) Payload() lorawan.PHYPayload {
	return p.payload
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (p rpacket) MarshalBinary() ([]byte, error) {
	var mtype byte
	switch p.payload.MHDR.MType {
	case lorawan.JoinRequest:
		fallthrough
	case lorawan.UnconfirmedDataUp:
		fallthrough
	case lorawan.ConfirmedDataUp:
		mtype = 1 // Up
	case lorawan.JoinAccept:
		fallthrough
	case lorawan.UnconfirmedDataDown:
		fallthrough
	case lorawan.ConfirmedDataDown:
		mtype = 2 // Down
	default:
		msg := fmt.Sprintf("Unsupported mtype: %s", p.payload.MHDR.MType.String())
		return nil, errors.New(errors.Implementation, msg)
	}

	dataMetadata, err := p.metadata.MarshalJSON()
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	dataPayload, err := p.payload.MarshalBinary()
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	rw := readwriter.New(nil)
	rw.Write([]byte{mtype})
	rw.Write(dataMetadata)
	rw.Write(dataPayload)
	return rw.Bytes()
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (p *rpacket) UnmarshalBinary(data []byte) error {
	if p == nil {
		return errors.New(errors.Structural, "Cannot unmarshal nil packet")
	}
	var isUp bool
	rw := readwriter.New(data)
	rw.Read(func(data []byte) {
		if data[0] == 1 {
			isUp = true
		}
	})

	var dataMetadata []byte
	rw.Read(func(data []byte) { dataMetadata = data })

	var dataPayload []byte
	rw.Read(func(data []byte) { dataPayload = data })

	if rw.Err() != nil {
		return errors.New(errors.Structural, rw.Err())
	}

	p.metadata = Metadata{}
	if err := p.metadata.UnmarshalJSON(dataMetadata); err != nil {
		return errors.New(errors.Structural, err)
	}

	p.payload = lorawan.NewPHYPayload(isUp)
	if err := p.payload.UnmarshalBinary(dataPayload); err != nil {
		return errors.New(errors.Structural, err)
	}

	return nil
}

// String implements the Stringer interface
func (p rpacket) String() string {
	str := "Packet {"
	str += fmt.Sprintf("\n\t%s}", p.metadata.String())
	str += fmt.Sprintf("\n\tPayload%+v\n}", p.payload)
	return str
}

type BPacket interface {
	Packet
	Commands() []lorawan.MACCommand
	DevEUI() lorawan.EUI64
	FCnt() uint32
	Metadata() Metadata
	Payload() []byte
	ValidateMIC(key lorawan.AES128Key) (bool, error)
}

// bpacket implements the core.BPacket interface
type bpacket struct {
	rpacket
}

// NewBPacket constructs a new broker packets given a payload and metadata
func NewBPacket(payload lorawan.PHYPayload, metadata Metadata) (BPacket, error) {
	packet, err := NewRPacket(payload, metadata)
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	macPayload := packet.Payload().MACPayload.(*lorawan.MACPayload)
	if len(macPayload.FRMPayload) != 1 {
		return nil, errors.New(errors.Structural, "Invalid frame payload. Expected exactly 1")
	}

	_, ok := macPayload.FRMPayload[0].(*lorawan.DataPayload)
	if !ok {
		return nil, errors.New(errors.Structural, "Invalid frame payload. Expected only data")
	}

	return bpacket{rpacket: packet.(rpacket)}, nil
}

// FCnt implements the core.BPacket interface
func (p bpacket) FCnt() uint32 {
	return p.payload.MACPayload.(*lorawan.MACPayload).FHDR.FCnt
}

// Payload implements the core.BPacket interface
func (p bpacket) Payload() []byte {
	macPayload := p.rpacket.payload.MACPayload.(*lorawan.MACPayload)
	return macPayload.FRMPayload[0].(*lorawan.DataPayload).Bytes
}

// ValidateMIC implements the core.BPacket interface
func (p bpacket) ValidateMIC(key lorawan.AES128Key) (bool, error) {
	return p.rpacket.payload.ValidateMIC(key)
}

// Commands implements the core.BPacket interface
func (p bpacket) Commands() []lorawan.MACCommand {
	return p.rpacket.payload.MACPayload.(*lorawan.MACPayload).FHDR.FOpts
}

type HPacket interface {
	Packet
	AppEUI() lorawan.EUI64
	DevEUI() lorawan.EUI64
	Payload() []byte    // FRMPayload
	Metadata() Metadata // TTL on down, DutyCycle + Rssi on Up
}

// hpacket implements the HPacket interface
type hpacket struct {
	appEUI   lorawan.EUI64
	devEUI   lorawan.EUI64
	payload  []byte
	metadata Metadata
}

// NewHPacket constructs a new Handler packet
func NewHPacket(a lorawan.EUI64, d lorawan.EUI64, p []byte, m Metadata) HPacket {
	if p == nil {
		p = make([]byte, 0)
	}
	return &hpacket{
		appEUI:   a,
		devEUI:   d,
		payload:  p,
		metadata: m,
	}
}

// AppEUI implements the core.HPacket interface
func (p hpacket) AppEUI() lorawan.EUI64 {
	return p.appEUI
}

// DevEUI implements the core.HPacket interface
func (p hpacket) DevEUI() lorawan.EUI64 {
	return p.devEUI
}

// Payload implements the core.HPacket interface
func (p hpacket) Payload() []byte {
	return p.payload
}

// Metadata implements the core.Metadata interface
func (p hpacket) Metadata() Metadata {
	return p.metadata
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (p hpacket) MarshalBinary() ([]byte, error) {
	dataMetadata, err := p.Metadata().MarshalJSON()
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	rw := readwriter.New(nil)
	rw.Write(p.appEUI)
	rw.Write(p.devEUI)
	rw.Write(p.payload)
	rw.Write(dataMetadata)
	return rw.Bytes()
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (p *hpacket) UnmarshalBinary(data []byte) error {
	if p == nil {
		return errors.New(errors.Structural, "Cannot unmarshal nil hpacket")
	}

	rw := readwriter.New(data)
	rw.Read(func(data []byte) { copy(p.appEUI[:], data) })
	rw.Read(func(data []byte) { copy(p.devEUI[:], data) })
	rw.Read(func(data []byte) { p.payload = data })
	var dataMetadata []byte
	rw.Read(func(data []byte) { dataMetadata = data })
	if err := p.metadata.UnmarshalJSON(dataMetadata); err != nil {
		return errors.New(errors.Structural, err)
	}
	return nil
}

// String implements the fmt.Stringer interface
func (p hpacket) String() string {
	str := "Packet {"
	str += fmt.Sprintf("\n\t%s}", p.metadata.String())
	str += fmt.Sprintf("\n\tAppEUI:%+x\n,", p.appEUI)
	str += fmt.Sprintf("\n\tDevEUI:%+x\n,", p.devEUI)
	str += fmt.Sprintf("\n\tPayload:%v\n}", p.Payload)
	return str
}
<commit_msg>[refactor] Write and implement APacket<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package refactor

import (
	"fmt"
	"io"

	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/readwriter"
	"github.com/brocaar/lorawan"
)

type RPacket interface {
	Packet
	Metadata() Metadata
	Payload() lorawan.PHYPayload
	DevEUI() lorawan.EUI64
}

// rpacket implements the core.RPacket interface
type rpacket struct {
	metadata Metadata
	payload  lorawan.PHYPayload
}

// NewRPacket construct a new router packet given a payload and metadata
func NewRPacket(payload lorawan.PHYPayload, metadata Metadata) (RPacket, error) {
	packet := rpacket{payload: payload, metadata: metadata}

	// Check and extract the devEUI
	if payload.MACPayload == nil {
		return nil, errors.New(errors.Structural, "MACPAyload should not be empty")
	}

	_, ok := payload.MACPayload.(*lorawan.MACPayload)
	if !ok {
		return nil, errors.New(errors.Structural, "Packet does not carry a MACPayload")
	}

	return &packet, nil
}

// DevEUI implements the core.BPacket interface
func (p rpacket) DevEUI() lorawan.EUI64 {
	var devEUI lorawan.EUI64
	copy(devEUI[4:], p.payload.MACPayload.(*lorawan.MACPayload).FHDR.DevAddr[:])
	return devEUI
}

// Metadata implements the core.RPacket interface
func (p rpacket) Metadata() Metadata {
	return p.metadata
}

// Payload implements the core.RPacket interface
func (p rpacket) Payload() lorawan.PHYPayload {
	return p.payload
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (p rpacket) MarshalBinary() ([]byte, error) {
	var mtype byte
	switch p.payload.MHDR.MType {
	case lorawan.JoinRequest:
		fallthrough
	case lorawan.UnconfirmedDataUp:
		fallthrough
	case lorawan.ConfirmedDataUp:
		mtype = 1 // Up
	case lorawan.JoinAccept:
		fallthrough
	case lorawan.UnconfirmedDataDown:
		fallthrough
	case lorawan.ConfirmedDataDown:
		mtype = 2 // Down
	default:
		msg := fmt.Sprintf("Unsupported mtype: %s", p.payload.MHDR.MType.String())
		return nil, errors.New(errors.Implementation, msg)
	}

	dataMetadata, err := p.metadata.MarshalJSON()
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	dataPayload, err := p.payload.MarshalBinary()
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	rw := readwriter.New(nil)
	rw.Write([]byte{mtype})
	rw.Write(dataMetadata)
	rw.Write(dataPayload)
	return rw.Bytes()
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (p *rpacket) UnmarshalBinary(data []byte) error {
	if p == nil {
		return errors.New(errors.Structural, "Cannot unmarshal nil packet")
	}
	var isUp bool
	rw := readwriter.New(data)
	rw.Read(func(data []byte) {
		if data[0] == 1 {
			isUp = true
		}
	})

	var dataMetadata []byte
	rw.Read(func(data []byte) { dataMetadata = data })

	var dataPayload []byte
	rw.Read(func(data []byte) { dataPayload = data })

	if rw.Err() != nil {
		return errors.New(errors.Structural, rw.Err())
	}

	p.metadata = Metadata{}
	if err := p.metadata.UnmarshalJSON(dataMetadata); err != nil {
		return errors.New(errors.Structural, err)
	}

	p.payload = lorawan.NewPHYPayload(isUp)
	if err := p.payload.UnmarshalBinary(dataPayload); err != nil {
		return errors.New(errors.Structural, err)
	}

	return nil
}

// String implements the Stringer interface
func (p rpacket) String() string {
	str := "Packet {"
	str += fmt.Sprintf("\n\t%s}", p.metadata.String())
	str += fmt.Sprintf("\n\tPayload%+v\n}", p.payload)
	return str
}

type BPacket interface {
	Packet
	Commands() []lorawan.MACCommand
	DevEUI() lorawan.EUI64
	FCnt() uint32
	Metadata() Metadata
	Payload() []byte
	ValidateMIC(key lorawan.AES128Key) (bool, error)
}

// bpacket implements the core.BPacket interface
type bpacket struct {
	rpacket
}

// NewBPacket constructs a new broker packets given a payload and metadata
func NewBPacket(payload lorawan.PHYPayload, metadata Metadata) (BPacket, error) {
	packet, err := NewRPacket(payload, metadata)
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	macPayload := packet.Payload().MACPayload.(*lorawan.MACPayload)
	if len(macPayload.FRMPayload) != 1 {
		return nil, errors.New(errors.Structural, "Invalid frame payload. Expected exactly 1")
	}

	_, ok := macPayload.FRMPayload[0].(*lorawan.DataPayload)
	if !ok {
		return nil, errors.New(errors.Structural, "Invalid frame payload. Expected only data")
	}

	return bpacket{rpacket: packet.(rpacket)}, nil
}

// FCnt implements the core.BPacket interface
func (p bpacket) FCnt() uint32 {
	return p.payload.MACPayload.(*lorawan.MACPayload).FHDR.FCnt
}

// Payload implements the core.BPacket interface
func (p bpacket) Payload() []byte {
	macPayload := p.rpacket.payload.MACPayload.(*lorawan.MACPayload)
	return macPayload.FRMPayload[0].(*lorawan.DataPayload).Bytes
}

// ValidateMIC implements the core.BPacket interface
func (p bpacket) ValidateMIC(key lorawan.AES128Key) (bool, error) {
	return p.rpacket.payload.ValidateMIC(key)
}

// Commands implements the core.BPacket interface
func (p bpacket) Commands() []lorawan.MACCommand {
	return p.rpacket.payload.MACPayload.(*lorawan.MACPayload).FHDR.FOpts
}

type HPacket interface {
	Packet
	AppEUI() lorawan.EUI64
	DevEUI() lorawan.EUI64
	Payload() []byte    // FRMPayload
	Metadata() Metadata // TTL on down, DutyCycle + Rssi on Up
}

// hpacket implements the HPacket interface
type hpacket struct {
	appEUI   lorawan.EUI64
	devEUI   lorawan.EUI64
	payload  []byte
	metadata Metadata
}

// NewHPacket constructs a new Handler packet
func NewHPacket(a lorawan.EUI64, d lorawan.EUI64, p []byte, m Metadata) HPacket {
	if p == nil {
		p = make([]byte, 0)
	}
	return &hpacket{
		appEUI:   a,
		devEUI:   d,
		payload:  p,
		metadata: m,
	}
}

// AppEUI implements the core.HPacket interface
func (p hpacket) AppEUI() lorawan.EUI64 {
	return p.appEUI
}

// DevEUI implements the core.HPacket interface
func (p hpacket) DevEUI() lorawan.EUI64 {
	return p.devEUI
}

// Payload implements the core.HPacket interface
func (p hpacket) Payload() []byte {
	return p.payload
}

// Metadata implements the core.Metadata interface
func (p hpacket) Metadata() Metadata {
	return p.metadata
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (p hpacket) MarshalBinary() ([]byte, error) {
	dataMetadata, err := p.Metadata().MarshalJSON()
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	rw := readwriter.New(nil)
	rw.Write(p.appEUI)
	rw.Write(p.devEUI)
	rw.Write(p.payload)
	rw.Write(dataMetadata)
	return rw.Bytes()
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (p *hpacket) UnmarshalBinary(data []byte) error {
	if p == nil {
		return errors.New(errors.Structural, "Cannot unmarshal nil hpacket")
	}

	rw := readwriter.New(data)
	rw.Read(func(data []byte) { copy(p.appEUI[:], data) })
	rw.Read(func(data []byte) { copy(p.devEUI[:], data) })
	rw.Read(func(data []byte) { p.payload = data })
	var dataMetadata []byte
	rw.Read(func(data []byte) { dataMetadata = data })
	if err := p.metadata.UnmarshalJSON(dataMetadata); err != nil {
		return errors.New(errors.Structural, err)
	}
	return nil
}

// String implements the fmt.Stringer interface
func (p hpacket) String() string {
	str := "Packet {"
	str += fmt.Sprintf("\n\t%s}", p.metadata.String())
	str += fmt.Sprintf("\n\tAppEUI:%+x\n,", p.appEUI)
	str += fmt.Sprintf("\n\tDevEUI:%+x\n,", p.devEUI)
	str += fmt.Sprintf("\n\tPayload:%v\n}", p.Payload)
	return str
}

type APacket interface {
	Packet
	Payload() []byte
	Metadata() []Metadata
}

// apacket implements the core.APacket interface
type apacket struct {
	payload  []byte
	metadata []Metadata
}

// NewAPacket constructs a new application packet
func NewAPacket(payload []byte, metadata []Metadata) (APacket, error) {
	if len(payload) == 0 {
		return nil, errors.New(errors.Structural, "Application packet must hold a payload")
	}

	return &apacket{payload: payload, metadata: metadata}, nil
}

// Payload implements the core.APacket interface
func (p apacket) Payload() []byte {
	return p.payload
}

// Metadata implements the core.Metadata interface
func (p apacket) Metadata() []Metadata {
	return p.metadata
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (p apacket) MarshalBinary() ([]byte, error) {
	rw := readwriter.New(nil)
	for _, m := range p.metadata {
		data, err := m.MarshalJSON()
		if err != nil {
			return nil, errors.New(errors.Structural, err)
		}
		rw.Write(data)
	}
	data, err := rw.Bytes()
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	rw = readwriter.New(nil)
	rw.Write(p.payload)
	rw.Write(data)

	return rw.Bytes()
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (p *apacket) UnmarshalBinary(data []byte) error {
	if p == nil {
		return errors.New(errors.Structural, "Cannot unmarshal nil apacket")
	}

	var dataMetadata []byte
	rw := readwriter.New(data)
	rw.Read(func(data []byte) { p.payload = data })
	rw.Read(func(data []byte) { dataMetadata = data })
	if rw.Err() != nil {
		return errors.New(errors.Structural, rw.Err())
	}

	p.metadata = make([]Metadata, 0)
	rw = readwriter.New(dataMetadata)
	for {
		var dataMetadata []byte
		rw.Read(func(data []byte) { dataMetadata = data })
		if rw.Err() != nil {
			err, ok := rw.Err().(errors.Failure)
			if ok && err.Fault == io.EOF {
				break
			}
			return errors.New(errors.Structural, rw.Err())
		}
		metadata := new(Metadata)
		if err := metadata.UnmarshalJSON(dataMetadata); err != nil {
			return errors.New(errors.Structural, err)
		}

		p.metadata = append(p.metadata, *metadata)
	}

	return nil
}

// String implements the fmt.Stringer interface
func (p apacket) String() string {
	return "TODO"
}
<|endoftext|>