<commit_before>package adyen

/**********
* Payment *
**********/

// One-click functionality gives the shopper the option to store their payment details with the merchant, within the Adyen environment.
//
// In this type of transaction, the shopper needs to enter the CVC code for the transaction to get through.
//
// Link: https://docs.adyen.com/developers/api-reference/payments-api#recurring
const (
	RecurringPaymentOneClick              = "ONECLICK"
	RecurringPaymentRecurring             = "RECURRING"
	ShopperInteractionContAuth            = "ContAuth"
	SelectRecurringDetailReferenceLatests = "LATEST"
)

// AuthoriseEncrypted structure for Authorisation request (with encrypted card information)
//
// Link - https://docs.adyen.com/developers/api-reference/payments-api#paymentrequest
type AuthoriseEncrypted struct {
	AdditionalData                   *AdditionalData `json:"additionalData,omitempty"`
	Amount                           *Amount         `json:"amount"`
	BillingAddress                   *Address        `json:"billingAddress,omitempty"`
	DeliveryAddress                  *Address        `json:"deliveryAddress,omitempty"`
	Reference                        string          `json:"reference"`
	MerchantAccount                  string          `json:"merchantAccount"`
	ShopperReference                 string          `json:"shopperReference,omitempty"` // Mandatory for recurring payment
	Recurring                        *Recurring      `json:"recurring,omitempty"`
	ShopperEmail                     string          `json:"shopperEmail,omitempty"`
	ShopperInteraction               string          `json:"shopperInteraction,omitempty"`
	ShopperIP                        string          `json:"shopperIP,omitempty"`
	ShopperLocale                    string          `json:"shopperLocale,omitempty"`
	ShopperName                      *Name           `json:"shopperName,omitempty"`
	SelectedRecurringDetailReference string          `json:"selectedRecurringDetailReference,omitempty"`
	BrowserInfo                      *BrowserInfo    `json:"browserInfo,omitempty"` // Required for a 3DS process
	CaptureDelayHours                *int            `json:"captureDelayHours,omitempty"`
}

// Authorise structure for Authorisation request (card is not encrypted)
//
// Link - https://docs.adyen.com/developers/api-reference/payments-api#paymentrequest
type Authorise struct {
	Card                             *Card        `json:"card,omitempty"`
	Amount                           *Amount      `json:"amount"`
	BillingAddress                   *Address     `json:"billingAddress,omitempty"`
	DeliveryAddress                  *Address     `json:"deliveryAddress,omitempty"`
	Reference                        string       `json:"reference"`
	MerchantAccount                  string       `json:"merchantAccount"`
	ShopperReference                 string       `json:"shopperReference,omitempty"` // Mandatory for recurring payment
	Recurring                        *Recurring   `json:"recurring,omitempty"`
	ShopperEmail                     string       `json:"shopperEmail,omitempty"`
	ShopperInteraction               string       `json:"shopperInteraction,omitempty"`
	ShopperIP                        string       `json:"shopperIP,omitempty"`
	ShopperLocale                    string       `json:"shopperLocale,omitempty"`
	ShopperName                      *Name        `json:"shopperName,omitempty"`
	SelectedRecurringDetailReference string       `json:"selectedRecurringDetailReference,omitempty"`
	BrowserInfo                      *BrowserInfo `json:"browserInfo,omitempty"` // Required for a 3DS process
	CaptureDelayHours                *int         `json:"captureDelayHours,omitempty"`
}

// AuthoriseResponse is a response structure for Adyen
//
// Link - https://docs.adyen.com/developers/api-reference/payments-api#paymentresult
type AuthoriseResponse struct {
	PspReference   string          `json:"pspReference"`
	ResultCode     string          `json:"resultCode"`
	AuthCode       string          `json:"authCode"`
	RefusalReason  string          `json:"refusalReason"`
	IssuerURL      string          `json:"issuerUrl"`
	MD             string          `json:"md"`
	PaRequest      string          `json:"paRequest"`
	AdditionalData *AdditionalData `json:"additionalData,omitempty"`
}

// AdditionalData stores encrypted information about customer's credit card
type AdditionalData struct {
	Content                           string      `json:"card.encrypted.json,omitempty"`
	AliasType                         string      `json:"aliasType,omitempty"`
	Alias                             string      `json:"alias,omitempty"`
	ExpiryDate                        string      `json:"expiryDate,omitempty"`
	CardBin                           string      `json:"cardBin,omitempty"`
	CardSummary                       string      `json:"cardSummary,omitempty"`
	PaymentMethod                     string      `json:"paymentMethod,omitempty"`
	CardPaymentMethod                 string      `json:"cardPaymentMethod,omitempty"`
	CardIssuingCountry                string      `json:"cardIssuingCountry,omitempty"`
	RecurringDetailReference          string      `json:"recurring.recurringDetailReference,omitempty"`
	ExecuteThreeD                     *StringBool `json:"executeThreeD,omitempty"`
	FundingSource                     string      `json:"fundingSource,omitempty"`
	RequestedTestAcquirerResponseCode int         `json:"RequestedTestAcquirerResponseCode,omitempty"`
}

// BrowserInfo hold information on the user browser
type BrowserInfo struct {
	AcceptHeader string `json:"acceptHeader"`
	UserAgent    string `json:"userAgent"`
}

// Recurring hold the behavior for a future payment : could be ONECLICK or RECURRING
type Recurring struct {
	Contract string `json:"contract"`
}

/*************
* Payment 3D *
*************/

// Authorise3D structure for Authorisation request (card is not encrypted)
//
// https://docs.adyen.com/developers/api-reference/payments-api#paymentrequest3d
type Authorise3D struct {
	BillingAddress  *Address     `json:"billingAddress,omitempty"`
	DeliveryAddress *Address     `json:"deliveryAddress,omitempty"`
	MD              string       `json:"md"`
	MerchantAccount string       `json:"merchantAccount"`
	BrowserInfo     *BrowserInfo `json:"browserInfo"`
	PaResponse      string       `json:"paResponse"`
	ShopperEmail    string       `json:"shopperEmail,omitempty"`
	ShopperIP       string       `json:"shopperIP,omitempty"`
	ShopperLocale   string       `json:"shopperLocale,omitempty"`
	ShopperName     *Name        `json:"shopperName,omitempty"`
}

/*******************
* Directory lookup *
*******************/

// DirectoryLookupRequest - get list of available payment methods based on skin, country and order details
//
// Description - https://docs.adyen.com/developers/api-reference/hosted-payment-pages-api#directoryrequest
// CountryCode could be used to test local payment methods, if client's IP is from different country
type DirectoryLookupRequest struct {
	CurrencyCode      string `url:"currencyCode"`
	MerchantAccount   string `url:"merchantAccount"`
	PaymentAmount     int    `url:"paymentAmount"`
	SkinCode          string `url:"skinCode"`
	MerchantReference string `url:"merchantReference"`
	SessionsValidity  string `url:"sessionValidity"`
	MerchantSig       string `url:"merchantSig"`
	CountryCode       string `url:"countryCode"`
	ShipBeforeDate    string `url:"shipBeforeDate"`
}

// DirectoryLookupResponse - api response for DirectoryLookupRequest
//
// Description - https://docs.adyen.com/developers/api-reference/hosted-payment-pages-api#directoryresponse
type DirectoryLookupResponse struct {
	PaymentMethods []PaymentMethod `json:"paymentMethods"`
}

// PaymentMethod - structure for single payment method in directory look up response
//
// Part of DirectoryLookupResponse
type PaymentMethod struct {
	BrandCode string   `json:"brandCode"`
	Name      string   `json:"name"`
	Logos     logos    `json:"logos"`
	Issuers   []issuer `json:"issuers"`
}

// logos - payment method logos
//
// Part of DirectoryLookupResponse
type logos struct {
	Normal string `json:"normal"`
	Small  string `json:"small"`
	Tiny   string `json:"tiny"`
}

// issuer - bank issuer type
//
// Part of DirectoryLookupResponse
type issuer struct {
	IssuerID string `json:"issuerId"`
	Name     string `json:"name"`
}

/***********
* Skip HPP *
***********/

// SkipHppRequest contains data that would be used to create Adyen HPP redirect URL
//
// Link: https://docs.adyen.com/developers/ecommerce-integration/local-payment-methods
//
// Request description: https://docs.adyen.com/developers/api-reference/hosted-payment-pages-api#skipdetailsrequest
type SkipHppRequest struct {
	MerchantReference string `url:"merchantReference"`
	PaymentAmount     int    `url:"paymentAmount"`
	CurrencyCode      string `url:"currencyCode"`
	ShipBeforeDate    string `url:"shipBeforeDate"`
	SkinCode          string `url:"skinCode"`
	MerchantAccount   string `url:"merchantAccount"`
	ShopperLocale     string `url:"shopperLocale"`
	SessionsValidity  string `url:"sessionValidity"`
	MerchantSig       string `url:"merchantSig"`
	CountryCode       string `url:"countryCode"`
	BrandCode         string `url:"brandCode"`
	IssuerID          string `url:"issuerId"`
}
<commit_msg>Add comment for trigger error from adyen<commit_after>package adyen

/**********
* Payment *
**********/

// One-click functionality gives the shopper the option to store their payment details with the merchant, within the Adyen environment.
//
// In this type of transaction, the shopper needs to enter the CVC code for the transaction to get through.
//
// Link: https://docs.adyen.com/developers/api-reference/payments-api#recurring
const (
	RecurringPaymentOneClick              = "ONECLICK"
	RecurringPaymentRecurring             = "RECURRING"
	ShopperInteractionContAuth            = "ContAuth"
	SelectRecurringDetailReferenceLatests = "LATEST"
)

// AuthoriseEncrypted structure for Authorisation request (with encrypted card information)
//
// Link - https://docs.adyen.com/developers/api-reference/payments-api#paymentrequest
type AuthoriseEncrypted struct {
	AdditionalData                   *AdditionalData `json:"additionalData,omitempty"`
	Amount                           *Amount         `json:"amount"`
	BillingAddress                   *Address        `json:"billingAddress,omitempty"`
	DeliveryAddress                  *Address        `json:"deliveryAddress,omitempty"`
	Reference                        string          `json:"reference"`
	MerchantAccount                  string          `json:"merchantAccount"`
	ShopperReference                 string          `json:"shopperReference,omitempty"` // Mandatory for recurring payment
	Recurring                        *Recurring      `json:"recurring,omitempty"`
	ShopperEmail                     string          `json:"shopperEmail,omitempty"`
	ShopperInteraction               string          `json:"shopperInteraction,omitempty"`
	ShopperIP                        string          `json:"shopperIP,omitempty"`
	ShopperLocale                    string          `json:"shopperLocale,omitempty"`
	ShopperName                      *Name           `json:"shopperName,omitempty"`
	SelectedRecurringDetailReference string          `json:"selectedRecurringDetailReference,omitempty"`
	BrowserInfo                      *BrowserInfo    `json:"browserInfo,omitempty"` // Required for a 3DS process
	CaptureDelayHours                *int            `json:"captureDelayHours,omitempty"`
}

// Authorise structure for Authorisation request (card is not encrypted)
//
// Link - https://docs.adyen.com/developers/api-reference/payments-api#paymentrequest
type Authorise struct {
	Card                             *Card        `json:"card,omitempty"`
	Amount                           *Amount      `json:"amount"`
	BillingAddress                   *Address     `json:"billingAddress,omitempty"`
	DeliveryAddress                  *Address     `json:"deliveryAddress,omitempty"`
	Reference                        string       `json:"reference"`
	MerchantAccount                  string       `json:"merchantAccount"`
	ShopperReference                 string       `json:"shopperReference,omitempty"` // Mandatory for recurring payment
	Recurring                        *Recurring   `json:"recurring,omitempty"`
	ShopperEmail                     string       `json:"shopperEmail,omitempty"`
	ShopperInteraction               string       `json:"shopperInteraction,omitempty"`
	ShopperIP                        string       `json:"shopperIP,omitempty"`
	ShopperLocale                    string       `json:"shopperLocale,omitempty"`
	ShopperName                      *Name        `json:"shopperName,omitempty"`
	SelectedRecurringDetailReference string       `json:"selectedRecurringDetailReference,omitempty"`
	BrowserInfo                      *BrowserInfo `json:"browserInfo,omitempty"` // Required for a 3DS process
	CaptureDelayHours                *int         `json:"captureDelayHours,omitempty"`
}

// AuthoriseResponse is a response structure for Adyen
//
// Link - https://docs.adyen.com/developers/api-reference/payments-api#paymentresult
type AuthoriseResponse struct {
	PspReference   string          `json:"pspReference"`
	ResultCode     string          `json:"resultCode"`
	AuthCode       string          `json:"authCode"`
	RefusalReason  string          `json:"refusalReason"`
	IssuerURL      string          `json:"issuerUrl"`
	MD             string          `json:"md"`
	PaRequest      string          `json:"paRequest"`
	AdditionalData *AdditionalData `json:"additionalData,omitempty"`
}

// AdditionalData stores encrypted information about customer's credit card
type AdditionalData struct {
	Content                           string      `json:"card.encrypted.json,omitempty"`
	AliasType                         string      `json:"aliasType,omitempty"`
	Alias                             string      `json:"alias,omitempty"`
	ExpiryDate                        string      `json:"expiryDate,omitempty"`
	CardBin                           string      `json:"cardBin,omitempty"`
	CardSummary                       string      `json:"cardSummary,omitempty"`
	PaymentMethod                     string      `json:"paymentMethod,omitempty"`
	CardPaymentMethod                 string      `json:"cardPaymentMethod,omitempty"`
	CardIssuingCountry                string      `json:"cardIssuingCountry,omitempty"`
	RecurringDetailReference          string      `json:"recurring.recurringDetailReference,omitempty"`
	ExecuteThreeD                     *StringBool `json:"executeThreeD,omitempty"`
	FundingSource                     string      `json:"fundingSource,omitempty"`
	RequestedTestAcquirerResponseCode int         `json:"RequestedTestAcquirerResponseCode,omitempty"` //Used for trigger error from adyen
}

// BrowserInfo hold information on the user browser
type BrowserInfo struct {
	AcceptHeader string `json:"acceptHeader"`
	UserAgent    string `json:"userAgent"`
}

// Recurring hold the behavior for a future payment : could be ONECLICK or RECURRING
type Recurring struct {
	Contract string `json:"contract"`
}

/*************
* Payment 3D *
*************/

// Authorise3D structure for Authorisation request (card is not encrypted)
//
// https://docs.adyen.com/developers/api-reference/payments-api#paymentrequest3d
type Authorise3D struct {
	BillingAddress  *Address     `json:"billingAddress,omitempty"`
	DeliveryAddress *Address     `json:"deliveryAddress,omitempty"`
	MD              string       `json:"md"`
	MerchantAccount string       `json:"merchantAccount"`
	BrowserInfo     *BrowserInfo `json:"browserInfo"`
	PaResponse      string       `json:"paResponse"`
	ShopperEmail    string       `json:"shopperEmail,omitempty"`
	ShopperIP       string       `json:"shopperIP,omitempty"`
	ShopperLocale   string       `json:"shopperLocale,omitempty"`
	ShopperName     *Name        `json:"shopperName,omitempty"`
}

/*******************
* Directory lookup *
*******************/

// DirectoryLookupRequest - get list of available payment methods based on skin, country and order details
//
// Description - https://docs.adyen.com/developers/api-reference/hosted-payment-pages-api#directoryrequest
// CountryCode could be used to test local payment methods, if client's IP is from different country
type DirectoryLookupRequest struct {
	CurrencyCode      string `url:"currencyCode"`
	MerchantAccount   string `url:"merchantAccount"`
	PaymentAmount     int    `url:"paymentAmount"`
	SkinCode          string `url:"skinCode"`
	MerchantReference string `url:"merchantReference"`
	SessionsValidity  string `url:"sessionValidity"`
	MerchantSig       string `url:"merchantSig"`
	CountryCode       string `url:"countryCode"`
	ShipBeforeDate    string `url:"shipBeforeDate"`
}

// DirectoryLookupResponse - api response for DirectoryLookupRequest
//
// Description - https://docs.adyen.com/developers/api-reference/hosted-payment-pages-api#directoryresponse
type DirectoryLookupResponse struct {
	PaymentMethods []PaymentMethod `json:"paymentMethods"`
}

// PaymentMethod - structure for single payment method in directory look up response
//
// Part of DirectoryLookupResponse
type PaymentMethod struct {
	BrandCode string   `json:"brandCode"`
	Name      string   `json:"name"`
	Logos     logos    `json:"logos"`
	Issuers   []issuer `json:"issuers"`
}

// logos - payment method logos
//
// Part of DirectoryLookupResponse
type logos struct {
	Normal string `json:"normal"`
	Small  string `json:"small"`
	Tiny   string `json:"tiny"`
}

// issuer - bank issuer type
//
// Part of DirectoryLookupResponse
type issuer struct {
	IssuerID string `json:"issuerId"`
	Name     string `json:"name"`
}

/***********
* Skip HPP *
***********/

// SkipHppRequest contains data that would be used to create Adyen HPP redirect URL
//
// Link: https://docs.adyen.com/developers/ecommerce-integration/local-payment-methods
//
// Request description: https://docs.adyen.com/developers/api-reference/hosted-payment-pages-api#skipdetailsrequest
type SkipHppRequest struct {
	MerchantReference string `url:"merchantReference"`
	PaymentAmount     int    `url:"paymentAmount"`
	CurrencyCode      string `url:"currencyCode"`
	ShipBeforeDate    string `url:"shipBeforeDate"`
	SkinCode          string `url:"skinCode"`
	MerchantAccount   string `url:"merchantAccount"`
	ShopperLocale     string `url:"shopperLocale"`
	SessionsValidity  string `url:"sessionValidity"`
	MerchantSig       string `url:"merchantSig"`
	CountryCode       string `url:"countryCode"`
	BrandCode         string `url:"brandCode"`
	IssuerID          string `url:"issuerId"`
}
<|endoftext|>


<commit_before>package pdb

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/TuftsBCB/seq"
	"github.com/TuftsBCB/structure"
)

type pdbParser struct {
	entry    *Entry
	curModel int
	line     []byte
	modified map[string]string
	seqres   map[byte][]string

	missing map[byte][]missingResidue
	r465    bool

	processed map[seen]bool
	lastSeen  seen
}

type missingResidue struct {
	residue string
	seqNum  int
	insCode byte
}

type seen struct {
	chain byte
	model int
}

func ReadPDB(fp string) (*Entry, error) {
	var reader io.Reader
	var err error

	f, err := os.Open(fp)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	reader = f

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fp) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	return Read(reader, fp)
}

func Read(reader io.Reader, fpath string) (*Entry, error) {
	entry := &Entry{
		Path:   fpath,
		Chains: make([]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	// Note that it is imperative that we preserve the order of ATOM records
	// as we read them. We are currently trying to replicate Fragbag, and this
	// is what Fragbag does. (A more stable approach would require more
	// information from the PDB file; like differentiating models, since
	// sorting on ATOM serial number isn't good enough.)
	breader := bufio.NewReaderSize(reader, 1000)
	parser := pdbParser{
		entry:     entry,
		curModel:  1,
		line:      nil,
		modified:  make(map[string]string),
		seqres:    make(map[byte][]string),
		missing:   make(map[byte][]missingResidue),
		r465:      false,
		processed: make(map[seen]bool),
		lastSeen:  seen{0, 0},
	}
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF && len(line) == 0 {
			break
		} else if err != io.EOF && err != nil {
			return nil, err
		}
		parser.line = line
		if err := parser.parseLine(); err != nil {
			return nil, err
		}
	}

	// We've got to back and translate SEQRES residues to their single-letter
	// abbreviations.
	// We didn't do this before, because the MODRES records typically come
	// afterwards. Ug.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.seqres[chain.Ident] {
			newr, err := parser.residueAbbrev(r)
			if err != nil {
				return nil, fmt.Errorf("PDB (SEQRES) '%s': %s",
					parser.entry.Path, err)
			}
			chain.Sequence = append(chain.Sequence, newr)
		}
	}

	// We also need to translate the missing residues.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.missing[chain.Ident] {
			abbrev, err := parser.residueAbbrev(r.residue)
			if err != nil {
				return nil, fmt.Errorf("PDB (REMARK 465) '%s': %s",
					parser.entry.Path, err)
			}
			newr := &Residue{
				Name:          abbrev,
				SequenceNum:   r.seqNum,
				InsertionCode: r.insCode,
				Atoms:         nil,
			}
			chain.Missing = append(chain.Missing, newr)
		}
	}

	// If we didn't pick up any chains, this probably isn't a valid PDB file.
	if len(entry.Chains) == 0 {
		return nil, fmt.Errorf("The file '%s' does not appear to be a valid "+
			"PDB file.", entry.Path)
	}

	// If we couldn't find an Id code, inspect the base name of the file path.
	if len(entry.IdCode) == 0 || entry.IdCode == "0000" {
		name := path.Base(entry.Path)
		switch {
		case len(name) >= 7 && name[0:3] == "pdb":
			entry.IdCode = name[3:7]
		case len(name) >= 6: // SCOP
			entry.Scop = stripExt(name)
			entry.IdCode = name[1:5]
		case len(name) == 7: // cath
			entry.Cath = stripExt(name)
			entry.IdCode = name[0:4]
		}
	}

	return entry, nil
}

func stripExt(s string) string {
	i := strings.Index(s, ".")
	if i == -1 {
		return s
	}
	return s[0:i]
}

func (p *pdbParser) parseLine() error {
	var err error

	switch p.cols(1, 6) {
	case "HEADER":
		p.entry.IdCode = p.cols(63, 66)
	case "MODEL":
		p.curModel, err = p.atoi("MODEL number [11-14]", 11, 14)
		if err != nil {
			return err
		}
	case "SEQRES":
		p.parseSeqres()
	case "MODRES":
		residue := p.cols(25, 27)
		if len(residue) == 0 {
			residue = "UNK"
		}
		p.modified[p.cols(13, 15)] = residue
	case "HET":
		residue := p.cols(8, 10)

		// Only add this if there isn't a MODRES for this residue.
		if _, ok := p.modified[residue]; !ok {
			p.modified[residue] = "UNK"
		}
	case "ATOM":
		fallthrough
	case "HETATM":
		if err := p.parseAtom(); err != nil {
			return err
		}
	case "TER":
		// Whatever ATOM record we saw last will have its {chain, model}
		// added to the 'processed' map. This effectively disallows any further
		// additions to this particular {chain, model}.
		// This cuts off HETATM's after "TER", but I'm fine with that for now.
		// fmt.Printf("%c %d\n", p.lastSeen.chain, p.lastSeen.model)
		p.processed[p.lastSeen] = true
	case "REMARK":
		num, err := p.atoi("REMARK identifier [8-10]", 8, 10)
		if err != nil {
			// Some people like to put in their own remarks and not conform
			// to the SPEC. Why............
			return nil
		}
		switch num {
		case 465:
			if err := p.parseRemark465(); err != nil {
				return err
			}
		}
	}
	return nil
}

func (p pdbParser) parseSeqres() {
	chain := p.getChain(p.at(12))
	if _, ok := p.seqres[chain.Ident]; !ok {
		p.seqres[chain.Ident] = make([]string, 0, 25)
	}
	for c := 20; c <= 68; c += 4 {
		res := p.cols(c, c+2)
		if len(res) == 0 {
			break
		}
		if chain.SeqType == -1 {
			// Label the sequence type if we haven't yet.
			// This basically just looks at the length of the unabbreviated
			// residue. len of 3 = protein, len of 2 = deoxy, len of 1 = ribo.
			chain.SeqType = getAbbrevType(res)
		}
		p.seqres[chain.Ident] = append(p.seqres[chain.Ident], res)
	}
}

func (p *pdbParser) parseAtom() error {
	ident := p.at(22)

	// If we've already processed this {chain, model}, then don't look
	// for more ATOMs.
	if p.processed[seen{ident, p.curModel}] {
		return nil
	}

	res := p.cols(18, 20)
	if res == "HOH" { // ignore water HETATMs
		return nil
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	seqNum, err := p.atoi("ATOM sequence number [23-26]", 23, 26)
	if err != nil {
		return err
	}

	// If this is a HETATM, we often see weird residues, so we'll suppress an
	// error if we don't recognize the residue. The residue will be set to 'X'.
	flexible := p.cols(1, 6) == "HETATM"
	residue, err := p.getResidue(ident, res, seqNum, insCode, flexible)
	if err != nil {
		return err
	}
	atom := Atom{
		Name:   p.cols(13, 16),
		Het:    p.cols(1, 6) == "HETATM",
		Coords: structure.Coords{},
	}

	atom.X, err = p.atof(31, 38)
	if err != nil {
		return err
	}
	atom.Y, err = p.atof(39, 46)
	if err != nil {
		return err
	}
	atom.Z, err = p.atof(47, 54)
	if err != nil {
		return err
	}

	p.lastSeen = seen{ident, p.curModel}
	residue.Atoms = append(residue.Atoms, atom)
	return nil
}

func (p *pdbParser) parseRemark465() error {
	residue := p.cols(16, 18)

	if residue == "RES" && p.at(20) == 'C' && p.cols(22, 27) == "SSSEQI" {
		p.r465 = true
		return nil
	}
	if !p.r465 {
		return nil
	}
	if p.r465 && (len(residue) == 0 || len(residue) > 3) {
		p.r465 = false
		return nil
	}

	chainIdent := p.at(20)

	seqNum, err := p.atoi("REMARK 465 sequence number [22-26]", 22, 26)
	if err != nil {
		return err
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	if _, ok := p.missing[chainIdent]; !ok {
		p.missing[chainIdent] = make([]missingResidue, 0)
	}
	missing := missingResidue{residue, seqNum, insCode}
	p.missing[chainIdent] = append(p.missing[chainIdent], missing)
	return nil
}

func (p *pdbParser) getChain(ident byte) *Chain {
	for i, chain := range p.entry.Chains {
		if chain.Ident == ident {
			return p.entry.Chains[i]
		}
	}
	chain := &Chain{
		Entry:    p.entry,
		Ident:    ident,
		SeqType:  -1,
		Sequence: make([]seq.Residue, 0, 25),
		Models:   make([]*Model, 0),
		Missing:  make([]*Residue, 0),
	}
	p.entry.Chains = append(p.entry.Chains, chain)
	return chain
}

func (p pdbParser) getModel(ident byte) *Model {
	chain := p.getChain(ident)
	for i, model := range chain.Models {
		if model.Num == p.curModel {
			return chain.Models[i]
		}
	}

	model := &Model{
		Entry:    p.entry,
		Chain:    chain,
		Num:      p.curModel,
		Residues: make([]*Residue, 0, 25),
	}
	chain.Models = append(chain.Models, model)
	return model
}

func (p pdbParser) getResidue(ident byte,
	res string, seqNum int, insCode byte, flexible bool) (*Residue, error) {

	model := p.getModel(ident)
	for i := range model.Residues {
		if model.Residues[i].SequenceNum == seqNum &&
			model.Residues[i].InsertionCode == insCode {

			return model.Residues[i], nil
		}
	}
	abbrev, err := p.residueAbbrev(res)
	if !flexible && err != nil {
		return nil, fmt.Errorf("PDB (ATOM) '%s': %s", p.entry.Path, err)
	}

	residue := &Residue{
		Name:          abbrev,
		SequenceNum:   seqNum,
		InsertionCode: insCode,
		Atoms:         make([]Atom, 0, 2),
	}
	model.Residues = append(model.Residues, residue)
	return residue, nil
}

func (p pdbParser) atoi(where string, start, end int) (int, error) {
	n, err := strconv.Atoi(p.cols(start, end))
	if err != nil {
		return 0, fmt.Errorf("Error in %s: %s", where, err)
	}
	return n, nil
}

func (p pdbParser) atof(start, end int) (float64, error) {
	return strconv.ParseFloat(p.cols(start, end), 64)
}

func (p pdbParser) cols(start, end int) string {
	rs, re := start-1, end
	if rs >= len(p.line) || rs < 0 {
		return ""
	}
	if re < 0 || re < rs {
		return ""
	}
	if re > len(p.line) {
		re = len(p.line)
	}
	return string(bytes.TrimSpace(p.line[rs:re]))
}

func (p pdbParser) at(column int) byte {
	i := column - 1
	if i < 0 || i >= len(p.line) {
		return 0
	}
	return p.line[i]
}

func (p pdbParser) residueAbbrev(long string) (seq.Residue, error) {
	if res, ok := p.modified[long]; ok {
		return getAbbrev(res)
	}
	abbrev, err := getAbbrev(long)

	// If we couldn't find a residue, check the missing residues. If this
	// residue shows up there, let's just ignore it and return 'X'.
	// If it isn't in the missing residues, then quit with an error.
	if err != nil {
		for _, missing := range p.missing {
			for _, residue := range missing {
				if residue.residue == long {
					return 'X', nil
				}
			}
		}
		return 'X', err
	}
	return abbrev, nil
}
<commit_msg>better debugging for integer parsing errors<commit_after>package pdb

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/TuftsBCB/seq"
	"github.com/TuftsBCB/structure"
)

type pdbParser struct {
	entry    *Entry
	curModel int
	line     []byte
	modified map[string]string
	seqres   map[byte][]string

	missing map[byte][]missingResidue
	r465    bool

	processed map[seen]bool
	lastSeen  seen
}

type missingResidue struct {
	residue string
	seqNum  int
	insCode byte
}

type seen struct {
	chain byte
	model int
}

func ReadPDB(fp string) (*Entry, error) {
	var reader io.Reader
	var err error

	f, err := os.Open(fp)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	reader = f

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fp) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	return Read(reader, fp)
}

func Read(reader io.Reader, fpath string) (*Entry, error) {
	entry := &Entry{
		Path:   fpath,
		Chains: make([]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	// Note that it is imperative that we preserve the order of ATOM records
	// as we read them. We are currently trying to replicate Fragbag, and this
	// is what Fragbag does. (A more stable approach would require more
	// information from the PDB file; like differentiating models, since
	// sorting on ATOM serial number isn't good enough.)
	breader := bufio.NewReaderSize(reader, 1000)
	parser := pdbParser{
		entry:     entry,
		curModel:  1,
		line:      nil,
		modified:  make(map[string]string),
		seqres:    make(map[byte][]string),
		missing:   make(map[byte][]missingResidue),
		r465:      false,
		processed: make(map[seen]bool),
		lastSeen:  seen{0, 0},
	}
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF && len(line) == 0 {
			break
		} else if err != io.EOF && err != nil {
			return nil, err
		}
		parser.line = line
		if err := parser.parseLine(); err != nil {
			return nil, err
		}
	}

	// We've got to back and translate SEQRES residues to their single-letter
	// abbreviations.
	// We didn't do this before, because the MODRES records typically come
	// afterwards. Ug.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.seqres[chain.Ident] {
			newr, err := parser.residueAbbrev(r)
			if err != nil {
				return nil, fmt.Errorf("PDB (SEQRES) '%s': %s",
					parser.entry.Path, err)
			}
			chain.Sequence = append(chain.Sequence, newr)
		}
	}

	// We also need to translate the missing residues.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.missing[chain.Ident] {
			abbrev, err := parser.residueAbbrev(r.residue)
			if err != nil {
				return nil, fmt.Errorf("PDB (REMARK 465) '%s': %s",
					parser.entry.Path, err)
			}
			newr := &Residue{
				Name:          abbrev,
				SequenceNum:   r.seqNum,
				InsertionCode: r.insCode,
				Atoms:         nil,
			}
			chain.Missing = append(chain.Missing, newr)
		}
	}

	// If we didn't pick up any chains, this probably isn't a valid PDB file.
	if len(entry.Chains) == 0 {
		return nil, fmt.Errorf("The file '%s' does not appear to be a valid "+
			"PDB file.", entry.Path)
	}

	// If we couldn't find an Id code, inspect the base name of the file path.
	if len(entry.IdCode) == 0 || entry.IdCode == "0000" {
		name := path.Base(entry.Path)
		switch {
		case len(name) >= 7 && name[0:3] == "pdb":
			entry.IdCode = name[3:7]
		case len(name) >= 6: // SCOP
			entry.Scop = stripExt(name)
			entry.IdCode = name[1:5]
		case len(name) == 7: // cath
			entry.Cath = stripExt(name)
			entry.IdCode = name[0:4]
		}
	}

	return entry, nil
}

func stripExt(s string) string {
	i := strings.Index(s, ".")
	if i == -1 {
		return s
	}
	return s[0:i]
}

func (p *pdbParser) parseLine() error {
	var err error

	switch p.cols(1, 6) {
	case "HEADER":
		p.entry.IdCode = p.cols(63, 66)
	case "MODEL":
		p.curModel, err = p.atoi("MODEL number [11-14]", 11, 14)
		if err != nil {
			return err
		}
	case "SEQRES":
		p.parseSeqres()
	case "MODRES":
		residue := p.cols(25, 27)
		if len(residue) == 0 {
			residue = "UNK"
		}
		p.modified[p.cols(13, 15)] = residue
	case "HET":
		residue := p.cols(8, 10)

		// Only add this if there isn't a MODRES for this residue.
		if _, ok := p.modified[residue]; !ok {
			p.modified[residue] = "UNK"
		}
	case "ATOM":
		fallthrough
	case "HETATM":
		if err := p.parseAtom(); err != nil {
			return err
		}
	case "TER":
		// Whatever ATOM record we saw last will have its {chain, model}
		// added to the 'processed' map. This effectively disallows any further
		// additions to this particular {chain, model}.
		// This cuts off HETATM's after "TER", but I'm fine with that for now.
		// fmt.Printf("%c %d\n", p.lastSeen.chain, p.lastSeen.model)
		p.processed[p.lastSeen] = true
	case "REMARK":
		num, err := p.atoi("REMARK identifier [8-10]", 8, 10)
		if err != nil {
			// Some people like to put in their own remarks and not conform
			// to the SPEC. Why............
			return nil
		}
		switch num {
		case 465:
			if err := p.parseRemark465(); err != nil {
				return err
			}
		}
	}
	return nil
}

func (p pdbParser) parseSeqres() {
	chain := p.getChain(p.at(12))
	if _, ok := p.seqres[chain.Ident]; !ok {
		p.seqres[chain.Ident] = make([]string, 0, 25)
	}
	for c := 20; c <= 68; c += 4 {
		res := p.cols(c, c+2)
		if len(res) == 0 {
			break
		}
		if chain.SeqType == -1 {
			// Label the sequence type if we haven't yet.
			// This basically just looks at the length of the unabbreviated
			// residue. len of 3 = protein, len of 2 = deoxy, len of 1 = ribo.
			chain.SeqType = getAbbrevType(res)
		}
		p.seqres[chain.Ident] = append(p.seqres[chain.Ident], res)
	}
}

func (p *pdbParser) parseAtom() error {
	ident := p.at(22)

	// If we've already processed this {chain, model}, then don't look
	// for more ATOMs.
	if p.processed[seen{ident, p.curModel}] {
		return nil
	}

	res := p.cols(18, 20)
	if res == "HOH" { // ignore water HETATMs
		return nil
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	seqNum, err := p.atoi("ATOM sequence number [23-26]", 23, 26)
	if err != nil {
		return err
	}

	// If this is a HETATM, we often see weird residues, so we'll suppress an
	// error if we don't recognize the residue. The residue will be set to 'X'.
	flexible := p.cols(1, 6) == "HETATM"
	residue, err := p.getResidue(ident, res, seqNum, insCode, flexible)
	if err != nil {
		return err
	}
	atom := Atom{
		Name:   p.cols(13, 16),
		Het:    p.cols(1, 6) == "HETATM",
		Coords: structure.Coords{},
	}

	atom.X, err = p.atof(31, 38)
	if err != nil {
		return err
	}
	atom.Y, err = p.atof(39, 46)
	if err != nil {
		return err
	}
	atom.Z, err = p.atof(47, 54)
	if err != nil {
		return err
	}

	p.lastSeen = seen{ident, p.curModel}
	residue.Atoms = append(residue.Atoms, atom)
	return nil
}

func (p *pdbParser) parseRemark465() error {
	residue := p.cols(16, 18)

	if residue == "RES" && p.at(20) == 'C' && p.cols(22, 27) == "SSSEQI" {
		p.r465 = true
		return nil
	}
	if !p.r465 {
		return nil
	}
	if p.r465 && (len(residue) == 0 || len(residue) > 3) {
		p.r465 = false
		return nil
	}

	chainIdent := p.at(20)

	seqNum, err := p.atoi("REMARK 465 sequence number [22-26]", 22, 26)
	if err != nil {
		return err
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	if _, ok := p.missing[chainIdent]; !ok {
		p.missing[chainIdent] = make([]missingResidue, 0)
	}
	missing := missingResidue{residue, seqNum, insCode}
	p.missing[chainIdent] = append(p.missing[chainIdent], missing)
	return nil
}

func (p *pdbParser) getChain(ident byte) *Chain {
	for i, chain := range p.entry.Chains {
		if chain.Ident == ident {
			return p.entry.Chains[i]
		}
	}
	chain := &Chain{
		Entry:    p.entry,
		Ident:    ident,
		SeqType:  -1,
		Sequence: make([]seq.Residue, 0, 25),
		Models:   make([]*Model, 0),
		Missing:  make([]*Residue, 0),
	}
	p.entry.Chains = append(p.entry.Chains, chain)
	return chain
}

func (p pdbParser) getModel(ident byte) *Model {
	chain := p.getChain(ident)
	for i, model := range chain.Models {
		if model.Num == p.curModel {
			return chain.Models[i]
		}
	}

	model := &Model{
		Entry:    p.entry,
		Chain:    chain,
		Num:      p.curModel,
		Residues: make([]*Residue, 0, 25),
	}
	chain.Models = append(chain.Models, model)
	return model
}

func (p pdbParser) getResidue(ident byte,
	res string, seqNum int, insCode byte, flexible bool) (*Residue, error) {

	model := p.getModel(ident)
	for i := range model.Residues {
		if model.Residues[i].SequenceNum == seqNum &&
			model.Residues[i].InsertionCode == insCode {

			return model.Residues[i], nil
		}
	}
	abbrev, err := p.residueAbbrev(res)
	if !flexible && err != nil {
		return nil, fmt.Errorf("PDB (ATOM) '%s': %s", p.entry.Path, err)
	}

	residue := &Residue{
		Name:          abbrev,
		SequenceNum:   seqNum,
		InsertionCode: insCode,
		Atoms:         make([]Atom, 0, 2),
	}
	model.Residues = append(model.Residues, residue)
	return residue, nil
}

func (p pdbParser) atoi(where string, start, end int) (int, error) {
	try := p.cols(start, end)
	n, err := strconv.Atoi(try)
	if err != nil {
		return 0, fmt.Errorf("Error in %s: %s (tried '%s')", where, err, try)
	}
	return n, nil
}

func (p pdbParser) atof(start, end int) (float64, error) {
	return strconv.ParseFloat(p.cols(start, end), 64)
}

func (p pdbParser) cols(start, end int) string {
	rs, re := start-1, end
	if rs >= len(p.line) || rs < 0 {
		return ""
	}
	if re < 0 || re < rs {
		return ""
	}
	if re > len(p.line) {
		re = len(p.line)
	}
	return string(bytes.TrimSpace(p.line[rs:re]))
}

func (p pdbParser) at(column int) byte {
	i := column - 1
	if i < 0 || i >= len(p.line) {
		return 0
	}
	return p.line[i]
}

func (p pdbParser) residueAbbrev(long string) (seq.Residue, error) {
	if res, ok := p.modified[long]; ok {
		return getAbbrev(res)
	}
	abbrev, err := getAbbrev(long)

	// If we couldn't find a residue, check the missing residues. If this
	// residue shows up there, let's just ignore it and return 'X'.
	// If it isn't in the missing residues, then quit with an error.
	if err != nil {
		for _, missing := range p.missing {
			for _, residue := range missing {
				if residue.residue == long {
					return 'X', nil
				}
			}
		}
		return 'X', err
	}
	return abbrev, nil
}
<|endoftext|>


<commit_before>package pdb

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"

	"github.com/BurntSushi/bcbgo/seq"
)

type pdbParser struct {
	entry    *Entry
	curModel int
	line     []byte
	modified map[string]string
	seqres   map[byte][]string

	missing map[byte][]missingResidue
	r465    bool

	processed map[seen]bool
	lastSeen  seen
}

type missingResidue struct {
	residue string
	seqNum  int
	insCode byte
}

type seen struct {
	chain byte
	model int
}

func ReadPDB(fp string) (*Entry, error) {
	var reader io.Reader
	var err error

	reader, err = os.Open(fp)
	if err != nil {
		return nil, err
	}

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fp) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	entry := &Entry{
		Path:   fp,
		Chains: make([]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	// Note that it is imperative that we preserve the order of ATOM records
	// as we read them. We are currently trying to replicate Fragbag, and this
	// is what Fragbag does. (A more stable approach would require more
	// information from the PDB file; like differentiating models, since
	// sorting on ATOM serial number isn't good enough.)
	breader := bufio.NewReaderSize(reader, 1000)
	parser := pdbParser{
		entry:     entry,
		curModel:  1,
		line:      nil,
		modified:  make(map[string]string),
		seqres:    make(map[byte][]string),
		missing:   make(map[byte][]missingResidue),
		r465:      false,
		processed: make(map[seen]bool),
		lastSeen:  seen{0, 0},
	}
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF && len(line) == 0 {
			break
		} else if err != io.EOF && err != nil {
			return nil, err
		}
		parser.line = line
		if err := parser.parseLine(); err != nil {
			return nil, err
		}
	}

	// We've got to back and translate SEQRES residues to their single-letter
	// abbreviations.
	// We didn't do this before, because the MODRES records typically come
	// afterwards. Ug.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.seqres[chain.Ident] {
			newr, err := parser.residueAbbrev(r)
			if err != nil {
				return nil, fmt.Errorf("PDB (SEQRES) '%s': %s",
					parser.entry.Path, err)
			}
			chain.Sequence = append(chain.Sequence, newr)
		}
	}

	// We also need to translate the missing residues.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.missing[chain.Ident] {
			abbrev, err := parser.residueAbbrev(r.residue)
			if err != nil {
				return nil, fmt.Errorf("PDB (REMARK 465) '%s': %s",
					parser.entry.Path, err)
			}
			newr := &Residue{
				Name:          abbrev,
				SequenceNum:   r.seqNum,
				InsertionCode: r.insCode,
				Atoms:         nil,
			}
			chain.Missing = append(chain.Missing, newr)
		}
	}

	// If we didn't pick up any chains, this probably isn't a valid PDB file.
	if len(entry.Chains) == 0 {
		return nil, fmt.Errorf("The file '%s' does not appear to be a valid "+
			"PDB file.", entry.Path)
	}

	// If we couldn't find an Id code, inspect the base name of the file path.
	if len(entry.IdCode) == 0 {
		name := path.Base(entry.Path)
		switch {
		case len(name) >= 7 && name[0:3] == "pdb":
			entry.IdCode = name[3:7]
		case len(name) == 7: // cath
			entry.IdCode = name[0:4]
		}
	}

	return entry, nil
}

func (p *pdbParser) parseLine() error {
	var err error

	switch p.cols(1, 6) {
	case "HEADER":
		p.entry.IdCode = p.cols(63, 66)
	case "MODEL":
		p.curModel, err = p.atoi(11, 14)
		if err != nil {
			return err
		}
	case "SEQRES":
		p.parseSeqres()
	case "MODRES":
		residue := p.cols(25, 27)
		if len(residue) == 0 {
			residue = "UNK"
		}
		p.modified[p.cols(13, 15)] = residue
	case "HET":
		residue := p.cols(8, 10)

		// Only add this if there isn't a MODRES for this residue.
		if _, ok := p.modified[residue]; !ok {
			p.modified[residue] = "UNK"
		}
	case "ATOM":
		fallthrough
	case "HETATM":
		if err := p.parseAtom(); err != nil {
			return err
		}
	case "TER":
		// Whatever ATOM record we saw last will have its {chain, model}
		// added to the 'processed' map. This effectively disallows any further
		// additions to this particular {chain, model}.
		// This cuts off HETATM's after "TER", but I'm fine with that for now.
		// fmt.Printf("%c %d\n", p.lastSeen.chain, p.lastSeen.model)
		p.processed[p.lastSeen] = true
	case "REMARK":
		num, err := p.atoi(8, 10)
		if err != nil {
			return err
		}
		switch num {
		case 465:
			if err := p.parseRemark465(); err != nil {
				return err
			}
		}
	}
	return nil
}

func (p pdbParser) parseSeqres() {
	chain := p.getChain(p.at(12))
	if _, ok := p.seqres[chain.Ident]; !ok {
		p.seqres[chain.Ident] = make([]string, 0, 25)
	}
	for c := 20; c <= 68; c += 4 {
		res := p.cols(c, c+2)
		if len(res) == 0 {
			break
		}
		if chain.SeqType == -1 {
			// Label the sequence type if we haven't yet.
			// This basically just looks at the length of the unabbreviated
			// residue. len of 3 = protein, len of 2 = deoxy, len of 1 = ribo.
			chain.SeqType = getAbbrevType(res)
		}
		p.seqres[chain.Ident] = append(p.seqres[chain.Ident], res)
	}
}

func (p *pdbParser) parseAtom() error {
	ident := p.at(22)

	// If we've already processed this {chain, model}, then don't look
	// for more ATOMs.
	if p.processed[seen{ident, p.curModel}] {
		return nil
	}

	res := p.cols(18, 20)
	if res == "HOH" { // ignore water HETATMs
		return nil
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	seqNum, err := p.atoi(23, 26)
	if err != nil {
		return err
	}

	// If this is a HETATM, we often see weird residues, so we'll suppress an
	// error if we don't recognize the residue. The residue will be set to 'X'.
	flexible := p.cols(1, 6) == "HETATM"
	residue, err := p.getResidue(ident, res, seqNum, insCode, flexible)
	if err != nil {
		return err
	}
	atom := Atom{
		Name:   p.cols(13, 16),
		Het:    p.cols(1, 6) == "HETATM",
		Coords: Coords{},
	}

	atom.X, err = p.atof(31, 38)
	if err != nil {
		return err
	}
	atom.Y, err = p.atof(39, 46)
	if err != nil {
		return err
	}
	atom.Z, err = p.atof(47, 54)
	if err != nil {
		return err
	}

	p.lastSeen = seen{ident, p.curModel}
	residue.Atoms = append(residue.Atoms, atom)
	return nil
}

func (p *pdbParser) parseRemark465() error {
	residue := p.cols(16, 18)

	if residue == "RES" && p.at(20) == 'C' && p.cols(22, 27) == "SSSEQI" {
		p.r465 = true
		return nil
	}
	if !p.r465 {
		return nil
	}
	if p.r465 && (len(residue) == 0 || len(residue) > 3) {
		p.r465 = false
		return nil
	}

	chainIdent := p.at(20)

	seqNum, err := p.atoi(22, 26)
	if err != nil {
		return err
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	if _, ok := p.missing[chainIdent]; !ok {
		p.missing[chainIdent] = make([]missingResidue, 0)
	}
	missing := missingResidue{residue, seqNum, insCode}
	p.missing[chainIdent] = append(p.missing[chainIdent], missing)
	return nil
}

func (p *pdbParser) getChain(ident byte) *Chain {
	for i, chain := range p.entry.Chains {
		if chain.Ident == ident {
			return p.entry.Chains[i]
		}
	}
	chain := &Chain{
		Entry:    p.entry,
		Ident:    ident,
		SeqType:  -1,
		Sequence: make([]seq.Residue, 0, 25),
		Models:   make([]*Model, 0),
		Missing:  make([]*Residue, 0),
	}
	p.entry.Chains = append(p.entry.Chains, chain)
	return chain
}

func (p pdbParser) getModel(ident byte) *Model {
	chain := p.getChain(ident)
	for i, model := range chain.Models {
		if model.Num == p.curModel {
			return chain.Models[i]
		}
	}

	model := &Model{
		Entry:    p.entry,
		Chain:    chain,
		Num:      p.curModel,
		Residues: make([]*Residue, 0, 25),
	}
	chain.Models = append(chain.Models, model)
	return model
}

func (p pdbParser) getResidue(ident byte,
	res string, seqNum int, insCode byte, flexible bool) (*Residue, error) {

	model := p.getModel(ident)
	for i := range model.Residues {
		if model.Residues[i].SequenceNum == seqNum &&
			model.Residues[i].InsertionCode == insCode {

			return model.Residues[i], nil
		}
	}
	abbrev, err := p.residueAbbrev(res)
	if !flexible && err != nil {
		return nil, fmt.Errorf("PDB (ATOM) '%s': %s", p.entry.Path, err)
	}

	residue := &Residue{
		Name:          abbrev,
		SequenceNum:   seqNum,
		InsertionCode: insCode,
		Atoms:         make([]Atom, 0, 2),
	}
	model.Residues = append(model.Residues, residue)
	return residue, nil
}

func (p pdbParser) atoi(start, end int) (int, error) {
	return strconv.Atoi(p.cols(start, end))
}

func (p pdbParser) atof(start, end int) (float64, error) {
	return strconv.ParseFloat(p.cols(start, end), 64)
}

func (p pdbParser) cols(start, end int) string {
	rs, re := start-1, end
	if rs >= len(p.line) || rs < 0 {
		return ""
	}
	if re > len(p.line) || re < 0 || re < rs {
		return ""
	}
	return string(bytes.TrimSpace(p.line[rs:re]))
}

func (p pdbParser) at(column int) byte {
	i := column - 1
	if i < 0 || i >= len(p.line) {
		return 0
	}
	return p.line[i]
}

func (p pdbParser) residueAbbrev(long string) (seq.Residue, error) {
	if res, ok := p.modified[long]; ok {
		return getAbbrev(res)
	}
	abbrev, err := getAbbrev(long)

	// If we couldn't find a residue, check the missing residues. If this
	// residue shows up there, let's just ignore it and return 'X'.
	// If it isn't in the missing residues, then quit with an error.
	if err != nil {
		for _, missing := range p.missing {
			for _, residue := range missing {
				if residue.residue == long {
					return 'X', nil
				}
			}
		}
		return 'X', err
	}
	return abbrev, nil
}
<commit_msg>Close open file handles please. Also, separate BOW db searching into 'bowseq' and 'bowpdb'. The former generates a fragment map using hhfrag, while the latter is the same as the old 'search-bowdb'.<commit_after>package pdb

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"

	"github.com/BurntSushi/bcbgo/seq"
)

type pdbParser struct {
	entry    *Entry
	curModel int
	line     []byte
	modified map[string]string
	seqres   map[byte][]string

	missing map[byte][]missingResidue
	r465    bool

	processed map[seen]bool
	lastSeen  seen
}

type missingResidue struct {
	residue string
	seqNum  int
	insCode byte
}

type seen struct {
	chain byte
	model int
}

func ReadPDB(fp string) (*Entry, error) {
	var reader io.Reader
	var err error

	f, err := os.Open(fp)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	reader = f

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fp) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	entry := &Entry{
		Path:   fp,
		Chains: make([]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	// Note that it is imperative that we preserve the order of ATOM records
	// as we read them. We are currently trying to replicate Fragbag, and this
	// is what Fragbag does. (A more stable approach would require more
	// information from the PDB file; like differentiating models, since
	// sorting on ATOM serial number isn't good enough.)
	breader := bufio.NewReaderSize(reader, 1000)
	parser := pdbParser{
		entry:     entry,
		curModel:  1,
		line:      nil,
		modified:  make(map[string]string),
		seqres:    make(map[byte][]string),
		missing:   make(map[byte][]missingResidue),
		r465:      false,
		processed: make(map[seen]bool),
		lastSeen:  seen{0, 0},
	}
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF && len(line) == 0 {
			break
		} else if err != io.EOF && err != nil {
			return nil, err
		}
		parser.line = line
		if err := parser.parseLine(); err != nil {
			return nil, err
		}
	}

	// We've got to back and translate SEQRES residues to their single-letter
	// abbreviations.
	// We didn't do this before, because the MODRES records typically come
	// afterwards. Ug.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.seqres[chain.Ident] {
			newr, err := parser.residueAbbrev(r)
			if err != nil {
				return nil, fmt.Errorf("PDB (SEQRES) '%s': %s",
					parser.entry.Path, err)
			}
			chain.Sequence = append(chain.Sequence, newr)
		}
	}

	// We also need to translate the missing residues.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.missing[chain.Ident] {
			abbrev, err := parser.residueAbbrev(r.residue)
			if err != nil {
				return nil, fmt.Errorf("PDB (REMARK 465) '%s': %s",
					parser.entry.Path, err)
			}
			newr := &Residue{
				Name:          abbrev,
				SequenceNum:   r.seqNum,
				InsertionCode: r.insCode,
				Atoms:         nil,
			}
			chain.Missing = append(chain.Missing, newr)
		}
	}

	// If we didn't pick up any chains, this probably isn't a valid PDB file.
	if len(entry.Chains) == 0 {
		return nil, fmt.Errorf("The file '%s' does not appear to be a valid "+
			"PDB file.", entry.Path)
	}

	// If we couldn't find an Id code, inspect the base name of the file path.
	if len(entry.IdCode) == 0 {
		name := path.Base(entry.Path)
		switch {
		case len(name) >= 7 && name[0:3] == "pdb":
			entry.IdCode = name[3:7]
		case len(name) == 7: // cath
			entry.IdCode = name[0:4]
		}
	}

	return entry, nil
}

func (p *pdbParser) parseLine() error {
	var err error

	switch p.cols(1, 6) {
	case "HEADER":
		p.entry.IdCode = p.cols(63, 66)
	case "MODEL":
		p.curModel, err = p.atoi(11, 14)
		if err != nil {
			return err
		}
	case "SEQRES":
		p.parseSeqres()
	case "MODRES":
		residue := p.cols(25, 27)
		if len(residue) == 0 {
			residue = "UNK"
		}
		p.modified[p.cols(13, 15)] = residue
	case "HET":
		residue := p.cols(8, 10)

		// Only add this if there isn't a MODRES for this residue.
		if _, ok := p.modified[residue]; !ok {
			p.modified[residue] = "UNK"
		}
	case "ATOM":
		fallthrough
	case "HETATM":
		if err := p.parseAtom(); err != nil {
			return err
		}
	case "TER":
		// Whatever ATOM record we saw last will have its {chain, model}
		// added to the 'processed' map. This effectively disallows any further
		// additions to this particular {chain, model}.
		// This cuts off HETATM's after "TER", but I'm fine with that for now.
		// fmt.Printf("%c %d\n", p.lastSeen.chain, p.lastSeen.model)
		p.processed[p.lastSeen] = true
	case "REMARK":
		num, err := p.atoi(8, 10)
		if err != nil {
			return err
		}
		switch num {
		case 465:
			if err := p.parseRemark465(); err != nil {
				return err
			}
		}
	}
	return nil
}

func (p pdbParser) parseSeqres() {
	chain := p.getChain(p.at(12))
	if _, ok := p.seqres[chain.Ident]; !ok {
		p.seqres[chain.Ident] = make([]string, 0, 25)
	}
	for c := 20; c <= 68; c += 4 {
		res := p.cols(c, c+2)
		if len(res) == 0 {
			break
		}
		if chain.SeqType == -1 {
			// Label the sequence type if we haven't yet.
			// This basically just looks at the length of the unabbreviated
			// residue. len of 3 = protein, len of 2 = deoxy, len of 1 = ribo.
			chain.SeqType = getAbbrevType(res)
		}
		p.seqres[chain.Ident] = append(p.seqres[chain.Ident], res)
	}
}

func (p *pdbParser) parseAtom() error {
	ident := p.at(22)

	// If we've already processed this {chain, model}, then don't look
	// for more ATOMs.
	if p.processed[seen{ident, p.curModel}] {
		return nil
	}

	res := p.cols(18, 20)
	if res == "HOH" { // ignore water HETATMs
		return nil
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	seqNum, err := p.atoi(23, 26)
	if err != nil {
		return err
	}

	// If this is a HETATM, we often see weird residues, so we'll suppress an
	// error if we don't recognize the residue. The residue will be set to 'X'.
	flexible := p.cols(1, 6) == "HETATM"
	residue, err := p.getResidue(ident, res, seqNum, insCode, flexible)
	if err != nil {
		return err
	}
	atom := Atom{
		Name:   p.cols(13, 16),
		Het:    p.cols(1, 6) == "HETATM",
		Coords: Coords{},
	}

	atom.X, err = p.atof(31, 38)
	if err != nil {
		return err
	}
	atom.Y, err = p.atof(39, 46)
	if err != nil {
		return err
	}
	atom.Z, err = p.atof(47, 54)
	if err != nil {
		return err
	}

	p.lastSeen = seen{ident, p.curModel}
	residue.Atoms = append(residue.Atoms, atom)
	return nil
}

func (p *pdbParser) parseRemark465() error {
	residue := p.cols(16, 18)

	if residue == "RES" && p.at(20) == 'C' && p.cols(22, 27) == "SSSEQI" {
		p.r465 = true
		return nil
	}
	if !p.r465 {
		return nil
	}
	if p.r465 && (len(residue) == 0 || len(residue) > 3) {
		p.r465 = false
		return nil
	}

	chainIdent := p.at(20)

	seqNum, err := p.atoi(22, 26)
	if err != nil {
		return err
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	if _, ok := p.missing[chainIdent]; !ok {
		p.missing[chainIdent] = make([]missingResidue, 0)
	}
	missing := missingResidue{residue, seqNum, insCode}
	p.missing[chainIdent] = append(p.missing[chainIdent], missing)
	return nil
}

func (p *pdbParser) getChain(ident byte) *Chain {
	for i, chain := range p.entry.Chains {
		if chain.Ident == ident {
			return p.entry.Chains[i]
		}
	}
	chain := &Chain{
		Entry:    p.entry,
		Ident:    ident,
		SeqType:  -1,
		Sequence: make([]seq.Residue, 0, 25),
		Models:   make([]*Model, 0),
		Missing:  make([]*Residue, 0),
	}
	p.entry.Chains = append(p.entry.Chains, chain)
	return chain
}

func (p pdbParser) getModel(ident byte) *Model {
	chain := p.getChain(ident)
	for i, model := range chain.Models {
		if model.Num == p.curModel {
			return chain.Models[i]
		}
	}

	model := &Model{
		Entry:    p.entry,
		Chain:    chain,
		Num:      p.curModel,
		Residues: make([]*Residue, 0, 25),
	}
	chain.Models = append(chain.Models, model)
	return model
}

func (p pdbParser) getResidue(ident byte,
	res string, seqNum int, insCode byte, flexible bool) (*Residue, error) {

	model := p.getModel(ident)
	for i := range model.Residues {
		if model.Residues[i].SequenceNum == seqNum &&
			model.Residues[i].InsertionCode == insCode {

			return model.Residues[i], nil
		}
	}
	abbrev, err := p.residueAbbrev(res)
	if !flexible && err != nil {
		return nil, fmt.Errorf("PDB (ATOM) '%s': %s", p.entry.Path, err)
	}

	residue := &Residue{
		Name:          abbrev,
		SequenceNum:   seqNum,
		InsertionCode: insCode,
		Atoms:         make([]Atom, 0, 2),
	}
	model.Residues = append(model.Residues, residue)
	return residue, nil
}

func (p pdbParser) atoi(start, end int) (int, error) {
	return strconv.Atoi(p.cols(start, end))
}

func (p pdbParser) atof(start, end int) (float64, error) {
	return strconv.ParseFloat(p.cols(start, end), 64)
}

func (p pdbParser) cols(start, end int) string {
	rs, re := start-1, end
	if rs >= len(p.line) || rs < 0 {
		return ""
	}
	if re > len(p.line) || re < 0 || re < rs {
		return ""
	}
	return string(bytes.TrimSpace(p.line[rs:re]))
}

func (p pdbParser) at(column int) byte {
	i := column - 1
	if i < 0 || i >= len(p.line) {
		return 0
	}
	return p.line[i]
}

func (p pdbParser) residueAbbrev(long string) (seq.Residue, error) {
	if res, ok := p.modified[long]; ok {
		return getAbbrev(res)
	}
	abbrev, err := getAbbrev(long)

	// If we couldn't find a residue, check the missing residues. If this
	// residue shows up there, let's just ignore it and return 'X'.
	// If it isn't in the missing residues, then quit with an error.
	if err != nil {
		for _, missing := range p.missing {
			for _, residue := range missing {
				if residue.residue == long {
					return 'X', nil
				}
			}
		}
		return 'X', err
	}
	return abbrev, nil
}
<|endoftext|>


<commit_before>package onlinestats

import "math"

func Pearson(a, b []float64) float64 {

	if len(a) != len(b) {
		panic("len(a) != len(b)")
	}

	var abar, bbar float64
	var n int
	for i := range a {
		if !math.IsNaN(a[i]) && !math.IsNaN(b[i]) {
			abar += a[i]
			bbar += b[i]
			n++
		}
	}
	nf := float64(n)
	abar, bbar = abar/nf, bbar/nf

	var numerator float64
	var sumAA, sumBB float64

	for i := range a {
		numerator += (a[i] - abar) * (b[i] - bbar)
		sumAA += (a[i] - abar) * (a[i] - abar)
		sumBB += (b[i] - bbar) * (b[i] - bbar)
	}

	return numerator / (math.Sqrt(sumAA) * math.Sqrt(sumBB))
}
<commit_msg>pearson: filter out more evil NaNs because they're pure evil.<commit_after>package onlinestats

import "math"

func Pearson(a, b []float64) float64 {

	if len(a) != len(b) {
		panic("len(a) != len(b)")
	}

	var abar, bbar float64
	var n int
	for i := range a {
		if !math.IsNaN(a[i]) && !math.IsNaN(b[i]) {
			abar += a[i]
			bbar += b[i]
			n++
		}
	}
	nf := float64(n)
	abar, bbar = abar/nf, bbar/nf

	var numerator float64
	var sumAA, sumBB float64

	for i := range a {
		if !math.IsNaN(a[i]) && !math.IsNaN(b[i]) {
			numerator += (a[i] - abar) * (b[i] - bbar)
			sumAA += (a[i] - abar) * (a[i] - abar)
			sumBB += (b[i] - bbar) * (b[i] - bbar)
		}
	}

	return numerator / (math.Sqrt(sumAA) * math.Sqrt(sumBB))
}
<|endoftext|>


<commit_before>// Copyright 2015 henrylee2cn Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package surfer

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

type (
	// Phantom 基于Phantomjs的下载器实现，作为surfer的补充
	// 效率较surfer会慢很多，但是因为模拟浏览器，破防性更好
	// 支持UserAgent/TryTimes/RetryPause/自定义js
	Phantom struct {
		PhantomjsFile string            //Phantomjs完整文件名
		TempJsDir     string            //临时js存放目录
		jsFileMap     map[string]string //已存在的js文件
	}
	// Response 用于解析Phantomjs的响应内容
	Response struct {
		Cookies []string
		Body    string
	}
)

// NewPhantom 创建一个Phantomjs下载器
func NewPhantom(phantomjsFile, tempJsDir string) Surfer {
	phantom := &Phantom{
		PhantomjsFile: phantomjsFile,
		TempJsDir:     tempJsDir,
		jsFileMap:     make(map[string]string),
	}
	if !filepath.IsAbs(phantom.PhantomjsFile) {
		phantom.PhantomjsFile, _ = filepath.Abs(phantom.PhantomjsFile)
	}
	if !filepath.IsAbs(phantom.TempJsDir) {
		phantom.TempJsDir, _ = filepath.Abs(phantom.TempJsDir)
	}
	// 创建/打开目录
	err := os.MkdirAll(phantom.TempJsDir, 0777)
	if err != nil {
		log.Printf("[E] Surfer: %v\n", err)
		return phantom
	}
	phantom.createJsFile("js", js)
	return phantom
}

// Download 实现surfer下载器接口
func (phantom *Phantom) Download(req *Request) (resp *http.Response, err error) {
	err = req.prepare()
	if err != nil {
		return resp, err
	}
	var encoding = "utf-8"
	if _, params, err := mime.ParseMediaType(req.Header.Get("Content-Type")); err == nil {
		if cs, ok := params["charset"]; ok {
			encoding = strings.ToLower(strings.TrimSpace(cs))
		}
	}

	req.Header.Del("Content-Type")

	var b, _ = req.ReadBody()

	resp = req.writeback(resp)

	var args = []string{
		phantom.jsFileMap["js"],
		req.Url,
		req.Header.Get("Cookie"),
		encoding,
		req.Header.Get("User-Agent"),
		string(b),
		strings.ToLower(req.Method),
		fmt.Sprint(int(req.PhantomTimeout / time.Millisecond)),
	}

	for i := 0; i < req.TryTimes; i++ {
		cmd := exec.Command(phantom.PhantomjsFile, args...)
		if resp.Body, err = cmd.StdoutPipe(); err != nil {
			time.Sleep(req.RetryPause)
			continue
		}
		err = cmd.Start()
		if err != nil || resp.Body == nil {
			time.Sleep(req.RetryPause)
			continue
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			time.Sleep(req.RetryPause)
			continue
		}
		retResp := Response{}
		err = json.Unmarshal(b, &retResp)
		if err != nil {
			time.Sleep(req.RetryPause)
			continue
		}
		resp.Header = req.Header
		delete(resp.Header, "Set-Cookie")
		for _, c := range retResp.Cookies {
			resp.Header.Add("Set-Cookie", c)
		}
		resp.Body = ioutil.NopCloser(strings.NewReader(retResp.Body))
		break
	}

	if err == nil {
		resp.StatusCode = http.StatusOK
		resp.Status = http.StatusText(http.StatusOK)
	} else {
		resp.StatusCode = http.StatusBadGateway
		resp.Status = err.Error()
	}
	return resp, err
}

// DestroyJsFiles 销毁js临时文件
func (phantom *Phantom) DestroyJsFiles() {
	p, _ := filepath.Split(phantom.TempJsDir)
	if p == "" {
		return
	}
	for _, filename := range phantom.jsFileMap {
		os.Remove(filename)
	}
	if len(WalkDir(p)) == 1 {
		os.Remove(p)
	}
}

func (phantom *Phantom) createJsFile(fileName, jsCode string) {
	fullFileName := filepath.Join(phantom.TempJsDir, fileName)
	// 创建并写入文件
	f, _ := os.Create(fullFileName)
	f.Write([]byte(jsCode))
	f.Close()
	phantom.jsFileMap[fileName] = fullFileName
}

/*
* system.args[0] == js
* system.args[1] == url
* system.args[2] == cookie
* system.args[3] == pageEncode
* system.args[4] == userAgent
* system.args[5] == postdata
* system.args[6] == method
* system.args[7] == timeout
 */
const js string = `
var system = require('system');
var page = require('webpage').create();
var url = system.args[1];
var cookie = system.args[2];
var pageEncode = system.args[3];
var userAgent = system.args[4];
var postdata = system.args[5];
var method = system.args[6];
var timeout = system.args[7];
var ret = "";
var exit = function() {
  console.log(ret);
  phantom.exit();
}

phantom.outputEncoding = pageEncode;
page.settings.userAgent = userAgent;
page.settings.resourceTimeout = timeout;
page.settings.XSSAuditingEnabled = true;
page.onResourceRequested = function(requestData, request) {
  request.setHeader('Cookie', cookie)
};
page.onError = function(msg, trace) {};
page.onResourceError = function(resourceError) {};
page.onLoadFinished = function(status) {
  if (status !== 'success') {
    console.log('Unable to access network');
  } else {
    var cookies = new Array();
    for (var i in page.cookies) {
      var cookie = page.cookies[i];
      var c = cookie["name"] + "=" + cookie["value"];
      for (var obj in cookie) {
        if (obj == 'name' || obj == 'value') {
          continue;
        }
        c += "; " + 　obj + "=" + cookie[obj];
      }
      cookies[i] = c;
    }

    var resp = {
      "Cookies": cookies,
      "Body": page.content
    };

    if (page.content.indexOf("div") != -1) {
      ret = JSON.stringify(resp);
      exit();
    }
  }
};

page.open(url, method, postdata, function(status) {});

if (timeout > 0) {
  setTimeout(function() {
    exit();
  }, timeout)
}
`
<commit_msg>1.phantomjs的timeout修改到onResourceTimeout 2.page.content.indexOf("div")改为page.content.indexOf("body")<commit_after>// Copyright 2015 henrylee2cn Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package surfer

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

type (
	// Phantom 基于Phantomjs的下载器实现，作为surfer的补充
	// 效率较surfer会慢很多，但是因为模拟浏览器，破防性更好
	// 支持UserAgent/TryTimes/RetryPause/自定义js
	Phantom struct {
		PhantomjsFile string            //Phantomjs完整文件名
		TempJsDir     string            //临时js存放目录
		jsFileMap     map[string]string //已存在的js文件
	}
	// Response 用于解析Phantomjs的响应内容
	Response struct {
		Cookies []string
		Body    string
	}
)

// NewPhantom 创建一个Phantomjs下载器
func NewPhantom(phantomjsFile, tempJsDir string) Surfer {
	phantom := &Phantom{
		PhantomjsFile: phantomjsFile,
		TempJsDir:     tempJsDir,
		jsFileMap:     make(map[string]string),
	}
	if !filepath.IsAbs(phantom.PhantomjsFile) {
		phantom.PhantomjsFile, _ = filepath.Abs(phantom.PhantomjsFile)
	}
	if !filepath.IsAbs(phantom.TempJsDir) {
		phantom.TempJsDir, _ = filepath.Abs(phantom.TempJsDir)
	}
	// 创建/打开目录
	err := os.MkdirAll(phantom.TempJsDir, 0777)
	if err != nil {
		log.Printf("[E] Surfer: %v\n", err)
		return phantom
	}
	phantom.createJsFile("js", js)
	return phantom
}

// Download 实现surfer下载器接口
func (phantom *Phantom) Download(req *Request) (resp *http.Response, err error) {
	err = req.prepare()
	if err != nil {
		return resp, err
	}
	var encoding = "utf-8"
	if _, params, err := mime.ParseMediaType(req.Header.Get("Content-Type")); err == nil {
		if cs, ok := params["charset"]; ok {
			encoding = strings.ToLower(strings.TrimSpace(cs))
		}
	}

	req.Header.Del("Content-Type")

	var b, _ = req.ReadBody()

	resp = req.writeback(resp)

	var args = []string{
		phantom.jsFileMap["js"],
		req.Url,
		req.Header.Get("Cookie"),
		encoding,
		req.Header.Get("User-Agent"),
		string(b),
		strings.ToLower(req.Method),
		fmt.Sprint(int(req.PhantomTimeout / time.Millisecond)),
	}

	for i := 0; i < req.TryTimes; i++ {
		cmd := exec.Command(phantom.PhantomjsFile, args...)
		if resp.Body, err = cmd.StdoutPipe(); err != nil {
			time.Sleep(req.RetryPause)
			continue
		}
		err = cmd.Start()
		if err != nil || resp.Body == nil {
			time.Sleep(req.RetryPause)
			continue
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Println("surfer read phantomjs out error:", err)
			time.Sleep(req.RetryPause)
			continue
		}
		retResp := Response{}
		err = json.Unmarshal(b, &retResp)
		if err != nil {
			log.Println("surfer phantomjs out:", string(b))
			time.Sleep(req.RetryPause)
			continue
		}
		resp.Header = req.Header
		delete(resp.Header, "Set-Cookie")
		for _, c := range retResp.Cookies {
			resp.Header.Add("Set-Cookie", c)
		}
		resp.Body = ioutil.NopCloser(strings.NewReader(retResp.Body))
		break
	}

	if err == nil {
		resp.StatusCode = http.StatusOK
		resp.Status = http.StatusText(http.StatusOK)
	} else {
		resp.StatusCode = http.StatusBadGateway
		resp.Status = err.Error()
	}
	return resp, err
}

// DestroyJsFiles 销毁js临时文件
func (phantom *Phantom) DestroyJsFiles() {
	p, _ := filepath.Split(phantom.TempJsDir)
	if p == "" {
		return
	}
	for _, filename := range phantom.jsFileMap {
		os.Remove(filename)
	}
	if len(WalkDir(p)) == 1 {
		os.Remove(p)
	}
}

func (phantom *Phantom) createJsFile(fileName, jsCode string) {
	fullFileName := filepath.Join(phantom.TempJsDir, fileName)
	// 创建并写入文件
	f, _ := os.Create(fullFileName)
	f.Write([]byte(jsCode))
	f.Close()
	phantom.jsFileMap[fileName] = fullFileName
}

/*
* system.args[0] == js
* system.args[1] == url
* system.args[2] == cookie
* system.args[3] == pageEncode
* system.args[4] == userAgent
* system.args[5] == postdata
* system.args[6] == method
* system.args[7] == timeout
 */
const js string = `
var system = require('system');
var page = require('webpage').create();
var url = system.args[1];
var cookie = system.args[2];
var pageEncode = system.args[3];
var userAgent = system.args[4];
var postdata = system.args[5];
var method = system.args[6];
var timeout = system.args[7];
var ret = "";
var exit = function () {
  console.log(ret);
  phantom.exit();
};

phantom.outputEncoding = pageEncode;
page.settings.userAgent = userAgent;
page.settings.resourceTimeout = timeout;
page.settings.XSSAuditingEnabled = true;
page.onResourceRequested = function (requestData, request) {
  request.setHeader('Cookie', cookie)
};
page.onError = function (msg, trace) {
  console.log("error:" + msg);
};
page.onResourceTimeout = function (e) {
  console.log("phantomjs onResourceTimeout error");
  // console.log(e.errorCode);   // it'll probably be 408
  // console.log(e.errorString); // it'll probably be 'Network timeout on resource'
  // console.log(e.url);         // the url whose request timed out
  phantom.exit(1);
};
page.onResourceError = function (resourceError) {
};
page.onLoadFinished = function (status) {
  if (status !== 'success') {
    console.log("phantomjs status:" + status);
    exit();
  } else {
    var cookies = new Array();
    for (var i in page.cookies) {
      var cookie = page.cookies[i];
      var c = cookie["name"] + "=" + cookie["value"];
      for (var obj in cookie) {
        if (obj == 'name' || obj == 'value') {
          continue;
        }
        c += "; " + obj + "=" + cookie[obj];
      }
      cookies[i] = c;
    }

    var resp = {
      "Cookies": cookies,
      "Body": page.content
    };

    if (page.content.indexOf("body") != -1) {
      ret = JSON.stringify(resp);
      exit();
    }
  }
};

page.open(url, method, postdata, function (status) {
});
`
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"
)

const (
	Version                = "0.9"
	SprocketsRequireClause = "//= require "
)

//FIXME: add test cases

var (
	outdir         = flag.String("outdir", filepath.Join("public", "assets"), "folder where to put packaged files")
	verbose        = flag.Bool("verbose", false, "turn on verbose logging")
	version        = flag.Bool("version", false, "print version and exit")
	digest         = flag.String("digest", "", "inject digest into output file names")
	generateDigest = flag.Bool("generatedigest", false, "generate digest from the output source")
	assetPath      = flag.String("assetpath", "", "set assetpath if your assets are not on the path specified in manifest")
	jsCompressor   = flag.String("jscompressor", "closure", "javascript compiler: closure or uglifyjs")
	// Platform specific stuff, will be configured in main
	shellForCommands      = ""
	shasumResultSeparator = ""
)

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("%v\n", Version)
		os.Exit(0)
	}

	// Parse manifest files
	manifestFiles := flag.Args()
	if len(manifestFiles) == 0 {
		fmt.Printf("Specify the manifest file(s) to process\n")
		os.Exit(1)
	}
	if *verbose {
		fmt.Printf("Manifest files: %v (%d)\n", manifestFiles, len(manifestFiles))
	}

	if runtime.GOOS == "windows" {
		shellForCommands = "sh"
		shasumResultSeparator = " *"
	} else {
		shellForCommands = "/bin/sh"
		shasumResultSeparator = "  "
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	// Create out dir (public/assets)
	if err := os.MkdirAll(*outdir, 0777); err != nil {
		panic(err)
	}

	// Validate js compiler
	if *jsCompressor != "closure" && *jsCompressor != "uglifyjs" {
		fmt.Printf("Invalid Javascript compiler: '%s'", jsCompressor)
		os.Exit(1)
	}

	// Create cache dir
	user, err := user.Current()
	if err != nil {
		panic(err)
	}
	appDir := filepath.Join(user.HomeDir, ".wortels")
	cacheDir := filepath.Join(appDir, "cache", *jsCompressor)
	if err := os.MkdirAll(cacheDir, 0777); err != nil {
		panic(err)
	}

	// Read in file list(s)
	files := make(map[string][]string)
	for _, manifest := range manifestFiles {
		b, err := ioutil.ReadFile(filepath.Join(*assetPath, manifest))
		if err != nil {
			panic(err)
		}
		for _, file := range strings.Split(string(b), "\n") {
			file = strings.TrimSpace(file)
			if file == "" {
				continue
			}
			// Convert Sprockets
			if strings.HasPrefix(file, SprocketsRequireClause) {
				file = strings.Replace(file, SprocketsRequireClause, "", 1)
				if !strings.HasSuffix(file, ".js") {
					file = file + ".js"
				}
			}
			// Ignore comments
			if strings.HasPrefix(file, "//") {
				continue
			}
			// Sprockets manifest files have relative paths
			file = filepath.Join(*assetPath, file)
			// Save the file under asset file list
			files[manifest] = append(files[manifest], file)
		}
	}
	if *verbose {
		fmt.Printf("File list: %v (%d)\n", files, len(files))
	}

	// Populate shasums dictionary
	shasums := make(map[string]string)
	dirs := make(map[string]bool)
	for manifest := range files {
		for _, file := range files[manifest] {
			dirs[filepath.Dir(file)] = true
		}
	}
	for dir, _ := range dirs {
		shasum(filepath.Join(dir, "*"), &shasums)
	}
	if *verbose {
		fmt.Printf("SHA database: %v (%d)\n", shasums, len(shasums))
	}

	// Find out which files need to be recompiled
	uniqueCompilationList := make(map[string]bool)
	for manifest := range files {
		for _, file := range files[manifest] {
			sha, knownFile := shasums[file]
			if !knownFile {
				panic(fmt.Sprintf("File not found in SHA1 database: '%v'\n", file))
			}
			cached := filepath.Join(cacheDir, sha)
			if !fileExists(cached) {
				uniqueCompilationList[file] = true
			}
		}
	}
	var compilationList []string
	for file := range uniqueCompilationList {
		compilationList = append(compilationList, file)
	}
	if *verbose {
		fmt.Printf("Files to compile: %v (%d)\n", compilationList, len(compilationList))
	}

	// Compile
	// http://closure-compiler.googlecode.com/files/compiler-latest.zip
	if len(compilationList) > 0 {
		compilationStart := time.Now()
		portableCompilationList := make([]string, len(compilationList))
		for i, path := range compilationList {
			portableCompilationList[i] = filepath.ToSlash(path)
		}
		cmd := jsCompileCommand(portableCompilationList, appDir)
		if *verbose {
			fmt.Printf("%v\n", cmd)
		}
		b, err := exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
		if err != nil {
			fmt.Printf("%v\n", string(b))
			os.Exit(1)
		}

		// Split compiler output into separate cached files
		var currentFile *os.File
		lines := strings.Split(string(b), "\n")
		lastLine := len(lines) - 1
		for i, line := range lines {
			if strings.HasPrefix(line, "// Input ") {
				if currentFile != nil {
					currentFile.Close()
				}
				i, err := strconv.Atoi(strings.Split(line, "// Input ")[1])
				if err != nil {
					panic(err)
				}
				file := compilationList[i]
				sha := shasums[file]
				cached := filepath.Join(cacheDir, sha)
				if *verbose {
					fmt.Printf("%v (%s)\n", file, sha)
				}
				currentFile, err = os.Create(cached)
				if err != nil {
					panic(err)
				}
				continue
			}
			if currentFile == nil {
				panic(fmt.Sprintf("No file to write to! Line: %s", line))
			}
			currentFile.Write([]byte(line))
			if i != lastLine {
				currentFile.Write([]byte("\n"))
			}
		}
		if currentFile != nil {
			currentFile.Close()
		}
		if *verbose {
			fmt.Printf("JS compiled in %v\n", time.Since(compilationStart))
		}
	}

	// Assemble asset file from compiled files
	var outputFiles []string
	for manifest := range files {
		catList := make([]string, len(files[manifest]))
		for i, file := range files[manifest] {
			catFile := filepath.Join(cacheDir, shasums[file])
			portableCatFile := filepath.ToSlash(catFile)
			catList[i] = portableCatFile
		}
		inputFiles := strings.Join(catList, " ")
		outputFile := filepath.Join(*outdir, filepath.Base(manifest))
		if *digest != "" {
			outputFile = injectDigest(outputFile, *digest)
		}
		portableOutputFile := filepath.ToSlash(outputFile)
		cmd := fmt.Sprintf("cat %s > %s", inputFiles, portableOutputFile)
		if *verbose {
			fmt.Printf("%v\n", cmd)
		}
		_, err = exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
		if err != nil {
			panic(err)
		}
		outputFiles = append(outputFiles, outputFile)
	}
	if *verbose {
		fmt.Printf("Output files: %v\n", outputFiles)
	}

	// Inject digests into output filenames, if no digest is given
	if *generateDigest {
		// Shasum the output files
		outputDigests := make(map[string]string)
		shasum(strings.Join(outputFiles, " "), &outputDigests)

		// Rename files to include the shasum
		for outputFile, sha1 := range outputDigests {
			renamedFile := injectDigest(outputFile, sha1)
			cmd := fmt.Sprintf("mv %s %s", outputFile, renamedFile)
			if *verbose {
				fmt.Printf("%v\n", cmd)
			}
			_, err = exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
			if err != nil {
				panic(err)
			}
		}
	}
}

func jsCompileCommand(portableCompilationList []string, appDir string) string {
	if *jsCompressor == "closure" {
		portableCompilerPath := filepath.ToSlash(filepath.Join(appDir, "compiler-latest", "compiler.jar"))
		return fmt.Sprintf("java -jar %s --warning_level QUIET --compilation_level SIMPLE_OPTIMIZATIONS --formatting print_input_delimiter --js %s",
			portableCompilerPath,
			strings.Join(portableCompilationList, " "))
	} else if *jsCompressor == "uglifyjs" {
		return fmt.Sprintf("uglifyjs %s", strings.Join(portableCompilationList, " "))
	}
	return ""
}

func injectDigest(outputFile, digest string) string {
	currentExt := filepath.Ext(outputFile)
	newExt := "-" + digest + currentExt
	return strings.Replace(outputFile, currentExt, newExt, -1)
}

func shasum(path string, shasums *map[string]string) {
	portablePath := filepath.ToSlash(path)
	cmd := fmt.Sprintf("shasum %s", portablePath)
	if *verbose {
		fmt.Printf("%v\n", cmd)
	}
	b, err := exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
	if err != nil {
		// newer shasum seems to exit status 1 even if there's no error?
		if err.Error() != "exit status 1" {
			panic(err)
		}
	}
	for _, shasumResult := range strings.Split(string(b), "\n") {
		if shasumResult != "" {
			// Ignore shasum messages a la
			// shasum: foo/bar: Is a directory
			if !strings.HasPrefix(shasumResult, "shasum: ") && !strings.HasSuffix(shasumResult, "Is a directory") {
				fields := strings.Split(shasumResult, shasumResultSeparator)

				if len(fields) < 2 {
					panic(fmt.Sprintf("Unexpected shasum result with separator '%s': '%v'\n", shasumResultSeparator, shasumResult))
				}
				sha := fields[0]
				file := filepath.FromSlash(fields[1])
				(*shasums)[file] = sha
			}
		}
	}
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			panic(err)
		}
	}
	return true
}
<commit_msg>FIXME-s<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"
)

const (
	Version                = "0.9"
	SprocketsRequireClause = "//= require "
)

//FIXME: add test cases

var (
	outdir         = flag.String("outdir", filepath.Join("public", "assets"), "folder where to put packaged files")
	verbose        = flag.Bool("verbose", false, "turn on verbose logging")
	version        = flag.Bool("version", false, "print version and exit")
	digest         = flag.String("digest", "", "inject digest into output file names")
	generateDigest = flag.Bool("generatedigest", false, "generate digest from the output source")
	assetPath      = flag.String("assetpath", "", "set assetpath if your assets are not on the path specified in manifest")
	// FIXME: use compiler name instead, like this: --compiler
	jsCompressor   = flag.String("jscompressor", "closure", "javascript compiler: closure or uglifyjs")
	// Platform specific stuff, will be configured in main
	shellForCommands      = ""
	shasumResultSeparator = ""
)

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("%v\n", Version)
		os.Exit(0)
	}

	// Parse manifest files
	manifestFiles := flag.Args()
	if len(manifestFiles) == 0 {
		fmt.Printf("Specify the manifest file(s) to process\n")
		os.Exit(1)
	}
	if *verbose {
		fmt.Printf("Manifest files: %v (%d)\n", manifestFiles, len(manifestFiles))
	}

	if runtime.GOOS == "windows" {
		shellForCommands = "sh"
		shasumResultSeparator = " *"
	} else {
		shellForCommands = "/bin/sh"
		shasumResultSeparator = "  "
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	// Create out dir (public/assets)
	if err := os.MkdirAll(*outdir, 0777); err != nil {
		panic(err)
	}

	// Validate js compiler
	if *jsCompressor != "closure" && *jsCompressor != "uglifyjs" {
		fmt.Printf("Invalid Javascript compiler: '%s'", jsCompressor)
		os.Exit(1)
	}

	// Create cache dir
	user, err := user.Current()
	if err != nil {
		panic(err)
	}
	appDir := filepath.Join(user.HomeDir, ".wortels")
	cacheDir := filepath.Join(appDir, "cache", *jsCompressor)
	if err := os.MkdirAll(cacheDir, 0777); err != nil {
		panic(err)
	}

	// Read in file list(s)
	files := make(map[string][]string)
	for _, manifest := range manifestFiles {
		b, err := ioutil.ReadFile(filepath.Join(*assetPath, manifest))
		if err != nil {
			panic(err)
		}
		for _, file := range strings.Split(string(b), "\n") {
			file = strings.TrimSpace(file)
			if file == "" {
				continue
			}
			// Convert Sprockets
			if strings.HasPrefix(file, SprocketsRequireClause) {
				file = strings.Replace(file, SprocketsRequireClause, "", 1)
				if !strings.HasSuffix(file, ".js") {
					file = file + ".js"
				}
			}
			// Ignore comments
			if strings.HasPrefix(file, "//") {
				continue
			}
			// Sprockets manifest files have relative paths
			file = filepath.Join(*assetPath, file)
			// Save the file under asset file list
			files[manifest] = append(files[manifest], file)
		}
	}
	if *verbose {
		fmt.Printf("File list: %v (%d)\n", files, len(files))
	}

	// Populate shasums dictionary
	shasums := make(map[string]string)
	dirs := make(map[string]bool)
	for manifest := range files {
		for _, file := range files[manifest] {
			dirs[filepath.Dir(file)] = true
		}
	}
	// FIXME: run concurrently
	for dir, _ := range dirs {
		shasum(filepath.Join(dir, "*"), &shasums)
	}
	if *verbose {
		fmt.Printf("SHA database: %v (%d)\n", shasums, len(shasums))
	}

	// Find out which files need to be recompiled
	uniqueCompilationList := make(map[string]bool)
	for manifest := range files {
		for _, file := range files[manifest] {
			sha, knownFile := shasums[file]
			if !knownFile {
				panic(fmt.Sprintf("File not found in SHA1 database: '%v'\n", file))
			}
			cached := filepath.Join(cacheDir, sha)
			if !fileExists(cached) {
				uniqueCompilationList[file] = true
			}
		}
	}
	var compilationList []string
	for file := range uniqueCompilationList {
		compilationList = append(compilationList, file)
	}
	if *verbose {
		fmt.Printf("Files to compile: %v (%d)\n", compilationList, len(compilationList))
	}

	// Compile
	// http://closure-compiler.googlecode.com/files/compiler-latest.zip
	if len(compilationList) > 0 {
		compilationStart := time.Now()
		portableCompilationList := make([]string, len(compilationList))
		for i, path := range compilationList {
			portableCompilationList[i] = filepath.ToSlash(path)
		}
		cmd := jsCompileCommand(portableCompilationList, appDir)
		if *verbose {
			fmt.Printf("%v\n", cmd)
		}
		b, err := exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
		if err != nil {
			fmt.Printf("%v\n", string(b))
			os.Exit(1)
		}

		// Split compiler output into separate cached files
		var currentFile *os.File
		lines := strings.Split(string(b), "\n")
		lastLine := len(lines) - 1
		for i, line := range lines {
			if strings.HasPrefix(line, "// Input ") {
				if currentFile != nil {
					currentFile.Close()
				}
				i, err := strconv.Atoi(strings.Split(line, "// Input ")[1])
				if err != nil {
					panic(err)
				}
				file := compilationList[i]
				sha := shasums[file]
				cached := filepath.Join(cacheDir, sha)
				if *verbose {
					fmt.Printf("%v (%s)\n", file, sha)
				}
				currentFile, err = os.Create(cached)
				if err != nil {
					panic(err)
				}
				continue
			}
			if currentFile == nil {
				panic(fmt.Sprintf("No file to write to! Line: %s", line))
			}
			currentFile.Write([]byte(line))
			if i != lastLine {
				currentFile.Write([]byte("\n"))
			}
		}
		if currentFile != nil {
			currentFile.Close()
		}
		if *verbose {
			fmt.Printf("JS compiled in %v\n", time.Since(compilationStart))
		}
	}

	// Assemble asset file from compiled files
	var outputFiles []string
	for manifest := range files {
		catList := make([]string, len(files[manifest]))
		for i, file := range files[manifest] {
			catFile := filepath.Join(cacheDir, shasums[file])
			portableCatFile := filepath.ToSlash(catFile)
			catList[i] = portableCatFile
		}
		inputFiles := strings.Join(catList, " ")
		outputFile := filepath.Join(*outdir, filepath.Base(manifest))
		if *digest != "" {
			outputFile = injectDigest(outputFile, *digest)
		}
		portableOutputFile := filepath.ToSlash(outputFile)
		cmd := fmt.Sprintf("cat %s > %s", inputFiles, portableOutputFile)
		if *verbose {
			fmt.Printf("%v\n", cmd)
		}
		_, err = exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
		if err != nil {
			panic(err)
		}
		outputFiles = append(outputFiles, outputFile)
	}
	if *verbose {
		fmt.Printf("Output files: %v\n", outputFiles)
	}

	// Inject digests into output filenames, if no digest is given
	if *generateDigest {
		// Shasum the output files
		outputDigests := make(map[string]string)
		shasum(strings.Join(outputFiles, " "), &outputDigests)

		// Rename files to include the shasum
		for outputFile, sha1 := range outputDigests {
			renamedFile := injectDigest(outputFile, sha1)
			cmd := fmt.Sprintf("mv %s %s", outputFile, renamedFile)
			if *verbose {
				fmt.Printf("%v\n", cmd)
			}
			_, err = exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
			if err != nil {
				panic(err)
			}
		}
	}
}

func jsCompileCommand(portableCompilationList []string, appDir string) string {
	if *jsCompressor == "closure" {
		portableCompilerPath := filepath.ToSlash(filepath.Join(appDir, "compiler-latest", "compiler.jar"))
		return fmt.Sprintf("java -jar %s --warning_level QUIET --compilation_level SIMPLE_OPTIMIZATIONS --formatting print_input_delimiter --js %s",
			portableCompilerPath,
			strings.Join(portableCompilationList, " "))
	} else if *jsCompressor == "uglifyjs" {
		return fmt.Sprintf("uglifyjs %s", strings.Join(portableCompilationList, " "))
	}
	return ""
}

func injectDigest(outputFile, digest string) string {
	currentExt := filepath.Ext(outputFile)
	newExt := "-" + digest + currentExt
	return strings.Replace(outputFile, currentExt, newExt, -1)
}

// FIXME: use go for reading files and sha hashing content.
func shasum(path string, shasums *map[string]string) {
	portablePath := filepath.ToSlash(path)
	cmd := fmt.Sprintf("shasum %s", portablePath)
	if *verbose {
		fmt.Printf("%v\n", cmd)
	}
	b, err := exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
	if err != nil {
		// newer shasum seems to exit status 1 even if there's no error?
		if err.Error() != "exit status 1" {
			panic(err)
		}
	}
	for _, shasumResult := range strings.Split(string(b), "\n") {
		if shasumResult != "" {
			// Ignore shasum messages a la
			// shasum: foo/bar: Is a directory
			if !strings.HasPrefix(shasumResult, "shasum: ") && !strings.HasSuffix(shasumResult, "Is a directory") {
				fields := strings.Split(shasumResult, shasumResultSeparator)

				if len(fields) < 2 {
					panic(fmt.Sprintf("Unexpected shasum result with separator '%s': '%v'\n", shasumResultSeparator, shasumResult))
				}
				sha := fields[0]
				file := filepath.FromSlash(fields[1])
				(*shasums)[file] = sha
			}
		}
	}
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			panic(err)
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"math"

	"github.com/ready-steady/hdf5"
	"github.com/ready-steady/statistics"
	"github.com/ready-steady/statistics/test"

	"../internal"
)

func main() {
	internal.Run(command)
}

func command(config *internal.Config, predict *hdf5.File, observe *hdf5.File) error {
	if predict == nil || observe == nil {
		return errors.New("two data files are required")
	}

	solution := new(internal.Solution)
	if err := predict.Get("solution", solution); err != nil {
		return err
	}

	observations := []float64{}
	if err := observe.Get("values", &observations); err != nil {
		return err
	}

	predictions := []float64{}
	if err := predict.Get("values", &predictions); err != nil {
		return err
	}

	ns := int(config.Assessment.Samples)
	no := len(observations) / ns
	nm := no / 2

	cut := func(data []float64, i int) []float64 {
		piece := make([]float64, ns)
		for j := 0; j < ns; j++ {
			piece[j] = data[j*no+i]
		}
		return piece
	}

	if config.Verbose {
		fmt.Println(solution)
	}

	an := make([]bool, nm)

	μo := make([]float64, nm)
	vo := make([]float64, nm)

	μp := make([]float64, nm)
	vp := make([]float64, nm)

	εμ := make([]float64, nm)
	εv := make([]float64, nm)
	εp := make([]float64, nm)

	εμr := make([]float64, nm)
	εvr := make([]float64, nm)

	analytic := len(solution.Expectation) == no

	// Compute errors across all outputs.
	for i := 0; i < nm; i++ {
		j := i * 2

		observations := cut(observations, j)
		predictions := cut(predictions, j)

		μo[i] = statistics.Mean(observations)
		vo[i] = statistics.Variance(observations)

		if analytic {
			μp[i] = solution.Expectation[j]
			vp[i] = solution.Expectation[j+1] - μp[i]*μp[i]
			if vp[i] < 0 {
				μp[i] = statistics.Mean(predictions)
				vp[i] = statistics.Variance(predictions)
			} else {
				an[i] = true
			}
		} else {
			μp[i] = statistics.Mean(predictions)
			vp[i] = statistics.Variance(predictions)
		}

		εμ[i] = math.Abs(μo[i] - μp[i])
		εv[i] = math.Abs(vo[i] - vp[i])

		εμr[i] = εμ[i] / μo[i]
		εvr[i] = εv[i] / vo[i]

		_, _, εp[i] = test.KolmogorovSmirnov(observations, predictions, 0)
	}

	if nm == 1 {
		fmt.Printf("Result: i %d, o %d, l %2d, n %5d, ",
			solution.Inputs, solution.Outputs, solution.Level, solution.Nodes)
		fmt.Printf("μ %.2e ± %.2e (%.2e), v %.2e ± %.2e (%.2e), p %.2e\n",
			μo[0], εμ[0], εμr[0], vo[0], εv[0], εvr[0], εp[0])

		return nil
	}

	fmt.Printf("Result: inputs %d, outputs %d, level %d, nodes %d\n",
		solution.Inputs, solution.Outputs, solution.Level, solution.Nodes)

	fmt.Printf("Outputs:\n")

	if config.Verbose {
		for i := 0; i < nm; i++ {
			fmt.Printf("%7d: μ %.2e ± %.2e (%.2e), v %.2e ± %.2e (%.2e), p %.2e",
				i, μo[i], εμ[i], εμr[i], vo[i], εv[i], εvr[i], εp[i])

			if !an[i] {
				fmt.Printf(" (sampling)")
			}

			fmt.Printf("\n")
		}
	}

	μμo, μεμ, μεμr := statistics.Mean(μo), statistics.Mean(εμ), statistics.Mean(clean(εμr))
	μvo, μεv, μεvr := statistics.Mean(vo), statistics.Mean(εv), statistics.Mean(clean(εvr))
	μεp := statistics.Mean(εp)

	fmt.Printf("Average: μ %.2e ± %.2e (%.2e), v %.2e ± %.2e (%.2e), p %.2e\n",
		μμo, μεμ, μεμr, μvo, μεv, μεvr, μεp)

	kμ, kv, kp := max(εμr), max(εvr), max(εp)

	fmt.Printf("Maximal: μ %.2e ± %.2e (%.2e), v %.2e ± %.2e (%.2e), p %.2e\n",
		μo[kμ], εμ[kμ], εμr[kμ], vo[kv], εv[kv], εvr[kv], εp[kp])

	return nil
}

func max(data []float64) int {
	value, k := math.Inf(-1), -1

	for i, x := range data {
		if !math.IsNaN(x) && !math.IsInf(x, 0) && x > value {
			value, k = x, i
		}
	}

	return k
}

func clean(data []float64) []float64 {
	result := make([]float64, 0, len(data))

	for _, x := range data {
		if !math.IsNaN(x) && !math.IsInf(x, 0) {
			result = append(result, x)
		}
	}

	return result
}
<commit_msg>cmd/compare: prioritize absolute errors<commit_after>package main

import (
	"errors"
	"fmt"
	"math"

	"github.com/ready-steady/hdf5"
	"github.com/ready-steady/statistics"
	"github.com/ready-steady/statistics/test"

	"../internal"
)

func main() {
	internal.Run(command)
}

func command(config *internal.Config, predict *hdf5.File, observe *hdf5.File) error {
	if predict == nil || observe == nil {
		return errors.New("two data files are required")
	}

	solution := new(internal.Solution)
	if err := predict.Get("solution", solution); err != nil {
		return err
	}

	observations := []float64{}
	if err := observe.Get("values", &observations); err != nil {
		return err
	}

	predictions := []float64{}
	if err := predict.Get("values", &predictions); err != nil {
		return err
	}

	ns := int(config.Assessment.Samples)
	no := len(observations) / ns
	nm := no / 2

	cut := func(data []float64, i int) []float64 {
		piece := make([]float64, ns)
		for j := 0; j < ns; j++ {
			piece[j] = data[j*no+i]
		}
		return piece
	}

	if config.Verbose {
		fmt.Println(solution)
	}

	an := make([]bool, nm)

	μo := make([]float64, nm)
	vo := make([]float64, nm)

	μp := make([]float64, nm)
	vp := make([]float64, nm)

	εμ := make([]float64, nm)
	εv := make([]float64, nm)
	εp := make([]float64, nm)

	εμr := make([]float64, nm)
	εvr := make([]float64, nm)

	analytic := len(solution.Expectation) == no

	// Compute errors across all outputs.
	for i := 0; i < nm; i++ {
		j := i * 2

		observations := cut(observations, j)
		predictions := cut(predictions, j)

		μo[i] = statistics.Mean(observations)
		vo[i] = statistics.Variance(observations)

		if analytic {
			μp[i] = solution.Expectation[j]
			vp[i] = solution.Expectation[j+1] - μp[i]*μp[i]
			if vp[i] < 0 {
				μp[i] = statistics.Mean(predictions)
				vp[i] = statistics.Variance(predictions)
			} else {
				an[i] = true
			}
		} else {
			μp[i] = statistics.Mean(predictions)
			vp[i] = statistics.Variance(predictions)
		}

		εμ[i] = math.Abs(μo[i] - μp[i])
		εv[i] = math.Abs(vo[i] - vp[i])

		εμr[i] = εμ[i] / μo[i]
		εvr[i] = εv[i] / vo[i]

		_, _, εp[i] = test.KolmogorovSmirnov(observations, predictions, 0)
	}

	if nm == 1 {
		fmt.Printf("Result: i %d, o %d, l %2d, n %5d, ",
			solution.Inputs, solution.Outputs, solution.Level, solution.Nodes)
		fmt.Printf("μ %.2e ± %.2e (%.2e), v %.2e ± %.2e (%.2e), p %.2e\n",
			μo[0], εμ[0], εμr[0], vo[0], εv[0], εvr[0], εp[0])

		return nil
	}

	fmt.Printf("Result: inputs %d, outputs %d, level %d, nodes %d\n",
		solution.Inputs, solution.Outputs, solution.Level, solution.Nodes)

	fmt.Printf("Outputs:\n")

	if config.Verbose {
		for i := 0; i < nm; i++ {
			fmt.Printf("%7d: μ %.2e ± %.2e (%.2e), v %.2e ± %.2e (%.2e), p %.2e",
				i, μo[i], εμ[i], εμr[i], vo[i], εv[i], εvr[i], εp[i])

			if !an[i] {
				fmt.Printf(" (sampling)")
			}

			fmt.Printf("\n")
		}
	}

	μμo, μεμ, μεμr := statistics.Mean(μo), statistics.Mean(εμ), statistics.Mean(clean(εμr))
	μvo, μεv, μεvr := statistics.Mean(vo), statistics.Mean(εv), statistics.Mean(clean(εvr))
	μεp := statistics.Mean(εp)

	fmt.Printf("Average: μ %.2e ± %.2e (%.2e), v %.2e ± %.2e (%.2e), p %.2e\n",
		μμo, μεμ, μεμr, μvo, μεv, μεvr, μεp)

	kμ, kv, kp := max(εμ), max(εv), max(εp)

	fmt.Printf("Maximal: μ %.2e ± %.2e (%.2e), v %.2e ± %.2e (%.2e), p %.2e\n",
		μo[kμ], εμ[kμ], εμr[kμ], vo[kv], εv[kv], εvr[kv], εp[kp])

	return nil
}

func max(data []float64) int {
	value, k := math.Inf(-1), -1

	for i, x := range data {
		if !math.IsNaN(x) && !math.IsInf(x, 0) && x > value {
			value, k = x, i
		}
	}

	return k
}

func clean(data []float64) []float64 {
	result := make([]float64, 0, len(data))

	for _, x := range data {
		if !math.IsNaN(x) && !math.IsInf(x, 0) {
			result = append(result, x)
		}
	}

	return result
}
<|endoftext|>