<commit_before>package main

import (
	"bytes"
	"context"
	"crypto"
	"fmt"
	"io"
	"log"

	"github.com/google/go-tpm-tools/cel"
	"github.com/google/go-tpm-tools/client"
	servpb "github.com/google/go-tpm-tools/launcher/proto/attestation_verifier/v0"
	pb "github.com/google/go-tpm-tools/proto/attest"
	"google.golang.org/grpc"
)

const defaultCELPCR = 13

var defaultCELHashAlgo = []crypto.Hash{crypto.SHA256, crypto.SHA1}

type tpmKeyFetcher func(rw io.ReadWriter) (*client.Key, error)
type principalIDTokenFetcher func(audience string) ([][]byte, error)

// AttestationAgent is an agent that interacts with GCE's Attestation Service
// to Verify an attestation message.
type AttestationAgent struct {
	tpm              io.ReadWriteCloser
	akFetcher        tpmKeyFetcher
	client           servpb.AttestationVerifierClient
	principalFetcher principalIDTokenFetcher
	cosCel           cel.CEL
}

// CreateAttestationAgent returns an agent capable of performing remote
// attestation using the machine's (v)TPM to GCE's Attestation Service.
// - tpm is a handle to the TPM on the instance
// - akFetcher is a func to fetch an attestation key: see go-tpm-tools/client.
// - conn is a client connection to the attestation service, typically created
//   `grpc.Dial`. It is the client's responsibility to close the connection.
func CreateAttestationAgent(tpm io.ReadWriteCloser, akFetcher tpmKeyFetcher, conn *grpc.ClientConn, principalFetcher principalIDTokenFetcher) *AttestationAgent {
	client := servpb.NewAttestationVerifierClient(conn)
	return &AttestationAgent{
		tpm:              tpm,
		client:           client,
		akFetcher:        akFetcher,
		principalFetcher: principalFetcher,
	}
}

// MeasureEvent takes in a cel.Content and append it to the CEL eventlog
// under the attesation agent.
func (a *AttestationAgent) MeasureEvent(event cel.Content) error {
	return a.cosCel.AppendEvent(a.tpm, defaultCELPCR, defaultCELHashAlgo, event)
}

// Attest fetches the nonce and connection ID from the Attestation Service,
// creates an attestation message, and returns the resultant
// principalIDTokens are Metadata Server-generated ID tokens for the instance.
func (a *AttestationAgent) Attest(ctx context.Context) ([]byte, error) {
	log.Println("Calling attestation verifier GetParams")
	params, err := a.client.GetParams(ctx, &servpb.GetParamsRequest{})
	if err != nil {
		return nil, fmt.Errorf("failed GetParams call: %v", err)
	}
	log.Println(params.String())

	principalTokens, err := a.principalFetcher(params.GetAudience())
	if err != nil {
		return nil, fmt.Errorf("failed to get principal tokens: %w", err)
	}

	attestation, err := a.getAttestation(params.GetNonce())
	if err != nil {
		return nil, err
	}

	req := &servpb.VerifyRequest{ConnId: params.GetConnId(), Attestation: attestation, PrincipalIdTokens: principalTokens}
	log.Println("Calling attestation verifier Verify")
	resp, err := a.client.Verify(ctx, req)
	if err != nil {
		return nil, fmt.Errorf("failed Verify call: %v", err)
	}
	return resp.GetClaimsToken(), nil
}

func (a *AttestationAgent) getAttestation(nonce []byte) (*pb.Attestation, error) {
	ak, err := a.akFetcher(a.tpm)
	if err != nil {
		return nil, fmt.Errorf("failed to get AK: %v", err)
	}
	defer ak.Close()

	var buf bytes.Buffer
	if err := a.cosCel.EncodeCEL(&buf); err != nil {
		return nil, err
	}

	attestation, err := ak.Attest(client.AttestOpts{Nonce: nonce, CanonicalEventLog: buf.Bytes()})
	if err != nil {
		return nil, fmt.Errorf("failed to attest: %v", err)
	}
	return attestation, nil
}
<commit_msg>Enable AK cert fetching<commit_after>package main

import (
	"bytes"
	"context"
	"crypto"
	"fmt"
	"io"
	"log"
	"net/http"

	"github.com/google/go-tpm-tools/cel"
	"github.com/google/go-tpm-tools/client"
	servpb "github.com/google/go-tpm-tools/launcher/proto/attestation_verifier/v0"
	pb "github.com/google/go-tpm-tools/proto/attest"
	"google.golang.org/grpc"
)

const defaultCELPCR = 13

var defaultCELHashAlgo = []crypto.Hash{crypto.SHA256, crypto.SHA1}

type tpmKeyFetcher func(rw io.ReadWriter) (*client.Key, error)
type principalIDTokenFetcher func(audience string) ([][]byte, error)

// AttestationAgent is an agent that interacts with GCE's Attestation Service
// to Verify an attestation message.
type AttestationAgent struct {
	tpm              io.ReadWriteCloser
	akFetcher        tpmKeyFetcher
	client           servpb.AttestationVerifierClient
	principalFetcher principalIDTokenFetcher
	cosCel           cel.CEL
}

// CreateAttestationAgent returns an agent capable of performing remote
// attestation using the machine's (v)TPM to GCE's Attestation Service.
// - tpm is a handle to the TPM on the instance
// - akFetcher is a func to fetch an attestation key: see go-tpm-tools/client.
// - conn is a client connection to the attestation service, typically created
//   `grpc.Dial`. It is the client's responsibility to close the connection.
func CreateAttestationAgent(tpm io.ReadWriteCloser, akFetcher tpmKeyFetcher, conn *grpc.ClientConn, principalFetcher principalIDTokenFetcher) *AttestationAgent {
	client := servpb.NewAttestationVerifierClient(conn)
	return &AttestationAgent{
		tpm:              tpm,
		client:           client,
		akFetcher:        akFetcher,
		principalFetcher: principalFetcher,
	}
}

// MeasureEvent takes in a cel.Content and append it to the CEL eventlog
// under the attesation agent.
func (a *AttestationAgent) MeasureEvent(event cel.Content) error {
	return a.cosCel.AppendEvent(a.tpm, defaultCELPCR, defaultCELHashAlgo, event)
}

// Attest fetches the nonce and connection ID from the Attestation Service,
// creates an attestation message, and returns the resultant
// principalIDTokens are Metadata Server-generated ID tokens for the instance.
func (a *AttestationAgent) Attest(ctx context.Context) ([]byte, error) {
	log.Println("Calling attestation verifier GetParams")
	params, err := a.client.GetParams(ctx, &servpb.GetParamsRequest{})
	if err != nil {
		return nil, fmt.Errorf("failed GetParams call: %v", err)
	}
	log.Println(params.String())

	principalTokens, err := a.principalFetcher(params.GetAudience())
	if err != nil {
		return nil, fmt.Errorf("failed to get principal tokens: %w", err)
	}

	attestation, err := a.getAttestation(params.GetNonce())
	if err != nil {
		return nil, err
	}

	req := &servpb.VerifyRequest{ConnId: params.GetConnId(), Attestation: attestation, PrincipalIdTokens: principalTokens}
	log.Println("Calling attestation verifier Verify")
	resp, err := a.client.Verify(ctx, req)
	if err != nil {
		return nil, fmt.Errorf("failed Verify call: %v", err)
	}
	return resp.GetClaimsToken(), nil
}

func (a *AttestationAgent) getAttestation(nonce []byte) (*pb.Attestation, error) {
	ak, err := a.akFetcher(a.tpm)
	if err != nil {
		return nil, fmt.Errorf("failed to get AK: %v", err)
	}
	defer ak.Close()

	var buf bytes.Buffer
	if err := a.cosCel.EncodeCEL(&buf); err != nil {
		return nil, err
	}

	attestation, err := ak.Attest(client.AttestOpts{Nonce: nonce, CanonicalEventLog: buf.Bytes(), CertChainFetcher: http.DefaultClient})
	if err != nil {
		return nil, fmt.Errorf("failed to attest: %v", err)
	}
	return attestation, nil
}
<|endoftext|>


<commit_before>package driver

import (
        "bytes"
        "encoding/json"
        "fmt"
        "log"
        "os"
        "os/exec"
        "regexp"
        "runtime"
        "strings"
        "syscall"
        "time"

        "github.com/hashicorp/nomad/client/config"
        "github.com/hashicorp/nomad/nomad/structs"
)

var (
        reRktVersion  = regexp.MustCompile("rkt version ([\\d\\.]+).+")
        reAppcVersion = regexp.MustCompile("appc version ([\\d\\.]+).+")
)

// RktDriver is a driver for running images via Rkt
// We attempt to chose sane defaults for now, with more configuration available
// planned in the future
type RktDriver struct {
        DriverContext
}

// rktHandle is returned from Start/Open as a handle to the PID
type rktHandle struct {
        proc   *os.Process
        name   string
        logger *log.Logger
        waitCh chan error
        doneCh chan struct{}
}

// rktPID is a struct to map the pid running the process to the vm image on
// disk
type rktPID struct {
        Pid  int
        Name string
}

// NewRktDriver is used to create a new exec driver
func NewRktDriver(ctx *DriverContext) Driver {
        return &RktDriver{*ctx}
}

func (d *RktDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
        // Only enable if we are root when running on non-windows systems.
        if runtime.GOOS != "windows" && syscall.Geteuid() != 0 {
                d.logger.Printf("[DEBUG] driver.rkt: must run as root user, disabling")
                return false, nil
        }

        outBytes, err := exec.Command("rkt", "version").Output()
        if err != nil {
                return false, nil
        }
        out := strings.TrimSpace(string(outBytes))

        rktMatches := reRktVersion.FindStringSubmatch(out)
        appcMatches := reRktVersion.FindStringSubmatch(out)
        if len(rktMatches) != 2 || len(appcMatches) != 2 {
                return false, fmt.Errorf("Unable to parse Rkt version string: %#v", rktMatches)
        }

        node.Attributes["driver.rkt"] = "true"
        node.Attributes["driver.rkt.version"] = rktMatches[0]
        node.Attributes["driver.rkt.appc.version"] = appcMatches[1]

        return true, nil
}

// Run an existing Rkt image.
func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
        trust_prefix, ok := task.Config["trust_prefix"]
        if !ok || trust_prefix == "" {
                return nil, fmt.Errorf("Missing trust prefix for rkt")
        }

        // Add the given trust prefix
        var outBuf, errBuf bytes.Buffer
        cmd := exec.Command("rkt", "trust", fmt.Sprintf("--prefix=%s", trust_prefix))
        cmd.Stdout = &outBuf
        cmd.Stderr = &errBuf
        d.logger.Printf("[DEBUG] Starting rkt command: %q", cmd.Args)
        if err := cmd.Run(); err != nil {
                return nil, fmt.Errorf(
                        "Error running rkt: %s\n\nOutput: %s\n\nError: %s",
                        err, outBuf.String(), errBuf.String())
        }
        d.logger.Printf("[DEBUG] Added trust prefix: %q", trust_prefix)

        name, ok := task.Config["name"]
        if !ok || name == "" {
                return nil, fmt.Errorf("Missing ACI name for rkt")
        }

        // Run the ACI
        var aoutBuf, aerrBuf bytes.Buffer
        acmd := exec.Command("rkt", "run", "--mds-register=false", "--interactive", name)
        acmd.Stdout = &aoutBuf
        acmd.Stderr = &aerrBuf
        d.logger.Printf("[DEBUG] Starting rkt command: %q", acmd.Args)
        if err := acmd.Run(); err != nil {
                return nil, fmt.Errorf(
                        "Error running rkt: %s\n\nOutput: %s\n\nError: %s",
                        err, aoutBuf.String(), aerrBuf.String())
        }
        d.logger.Printf("[DEBUG] Started ACI: %q", name)
        h := &rktHandle{
                proc:   acmd.Process,
                name:   name,
                logger: d.logger,
                doneCh: make(chan struct{}),
                waitCh: make(chan error, 1),
        }
        go h.run()
        return h, nil
}

func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
        // Parse the handle
        pidBytes := []byte(strings.TrimPrefix(handleID, "Rkt:"))
        qpid := &rktPID{}
        if err := json.Unmarshal(pidBytes, qpid); err != nil {
                return nil, fmt.Errorf("failed to parse Rkt handle '%s': %v", handleID, err)
        }

        // Find the process
        proc, err := os.FindProcess(qpid.Pid)
        if proc == nil || err != nil {
                return nil, fmt.Errorf("failed to find Rkt PID %d: %v", qpid.Pid, err)
        }

        // Return a driver handle
        h := &rktHandle{
                proc:   proc,
                name:   qpid.Name,
                logger: d.logger,
                doneCh: make(chan struct{}),
                waitCh: make(chan error, 1),
        }

        go h.run()
        return h, nil
}

func (h *rktHandle) ID() string {
        // Return a handle to the PID
        pid := &rktPID{
                Pid:  h.proc.Pid,
                Name: h.name,
        }
        data, err := json.Marshal(pid)
        if err != nil {
                h.logger.Printf("[ERR] failed to marshal rkt PID to JSON: %s", err)
        }
        return fmt.Sprintf("Rkt:%s", string(data))
}

func (h *rktHandle) WaitCh() chan error {
        return h.waitCh
}

func (h *rktHandle) Update(task *structs.Task) error {
        // Update is not possible
        return nil
}

// Kill is used to terminate the task. We send an Interrupt
// and then provide a 5 second grace period before doing a Kill.
func (h *rktHandle) Kill() error {
        h.proc.Signal(os.Interrupt)
        select {
        case <-h.doneCh:
                return nil
        case <-time.After(5 * time.Second):
                return h.proc.Kill()
        }
}

func (h *rktHandle) run() {
        ps, err := h.proc.Wait()
        close(h.doneCh)
        if err != nil {
                h.waitCh <- err
        } else if !ps.Success() {
                h.waitCh <- fmt.Errorf("task exited with error")
        }
        close(h.waitCh)
}
<commit_msg>Do not run containers interactively<commit_after>package driver

import (
        "bytes"
        "encoding/json"
        "fmt"
        "log"
        "os"
        "os/exec"
        "regexp"
        "runtime"
        "strings"
        "syscall"
        "time"

        "github.com/hashicorp/nomad/client/config"
        "github.com/hashicorp/nomad/nomad/structs"
)

var (
        reRktVersion  = regexp.MustCompile("rkt version ([\\d\\.]+).+")
        reAppcVersion = regexp.MustCompile("appc version ([\\d\\.]+).+")
)

// RktDriver is a driver for running images via Rkt
// We attempt to chose sane defaults for now, with more configuration available
// planned in the future
type RktDriver struct {
        DriverContext
}

// rktHandle is returned from Start/Open as a handle to the PID
type rktHandle struct {
        proc   *os.Process
        name   string
        logger *log.Logger
        waitCh chan error
        doneCh chan struct{}
}

// rktPID is a struct to map the pid running the process to the vm image on
// disk
type rktPID struct {
        Pid  int
        Name string
}

// NewRktDriver is used to create a new exec driver
func NewRktDriver(ctx *DriverContext) Driver {
        return &RktDriver{*ctx}
}

func (d *RktDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
        // Only enable if we are root when running on non-windows systems.
        if runtime.GOOS != "windows" && syscall.Geteuid() != 0 {
                d.logger.Printf("[DEBUG] driver.rkt: must run as root user, disabling")
                return false, nil
        }

        outBytes, err := exec.Command("rkt", "version").Output()
        if err != nil {
                return false, nil
        }
        out := strings.TrimSpace(string(outBytes))

        rktMatches := reRktVersion.FindStringSubmatch(out)
        appcMatches := reRktVersion.FindStringSubmatch(out)
        if len(rktMatches) != 2 || len(appcMatches) != 2 {
                return false, fmt.Errorf("Unable to parse Rkt version string: %#v", rktMatches)
        }

        node.Attributes["driver.rkt"] = "true"
        node.Attributes["driver.rkt.version"] = rktMatches[0]
        node.Attributes["driver.rkt.appc.version"] = appcMatches[1]

        return true, nil
}

// Run an existing Rkt image.
func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
        trust_prefix, ok := task.Config["trust_prefix"]
        if !ok || trust_prefix == "" {
                return nil, fmt.Errorf("Missing trust prefix for rkt")
        }

        // Add the given trust prefix
        var outBuf, errBuf bytes.Buffer
        cmd := exec.Command("rkt", "trust", fmt.Sprintf("--prefix=%s", trust_prefix))
        cmd.Stdout = &outBuf
        cmd.Stderr = &errBuf
        d.logger.Printf("[DEBUG] Starting rkt command: %q", cmd.Args)
        if err := cmd.Run(); err != nil {
                return nil, fmt.Errorf(
                        "Error running rkt: %s\n\nOutput: %s\n\nError: %s",
                        err, outBuf.String(), errBuf.String())
        }
        d.logger.Printf("[DEBUG] Added trust prefix: %q", trust_prefix)

        name, ok := task.Config["name"]
        if !ok || name == "" {
                return nil, fmt.Errorf("Missing ACI name for rkt")
        }

        // Run the ACI
        var aoutBuf, aerrBuf bytes.Buffer
        acmd := exec.Command("rkt", "run", "--mds-register=false", name)
        acmd.Stdout = &aoutBuf
        acmd.Stderr = &aerrBuf
        d.logger.Printf("[DEBUG] Starting rkt command: %q", acmd.Args)
        if err := acmd.Run(); err != nil {
                return nil, fmt.Errorf(
                        "Error running rkt: %s\n\nOutput: %s\n\nError: %s",
                        err, aoutBuf.String(), aerrBuf.String())
        }
        d.logger.Printf("[DEBUG] Started ACI: %q", name)
        h := &rktHandle{
                proc:   acmd.Process,
                name:   name,
                logger: d.logger,
                doneCh: make(chan struct{}),
                waitCh: make(chan error, 1),
        }
        go h.run()
        return h, nil
}

func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
        // Parse the handle
        pidBytes := []byte(strings.TrimPrefix(handleID, "Rkt:"))
        qpid := &rktPID{}
        if err := json.Unmarshal(pidBytes, qpid); err != nil {
                return nil, fmt.Errorf("failed to parse Rkt handle '%s': %v", handleID, err)
        }

        // Find the process
        proc, err := os.FindProcess(qpid.Pid)
        if proc == nil || err != nil {
                return nil, fmt.Errorf("failed to find Rkt PID %d: %v", qpid.Pid, err)
        }

        // Return a driver handle
        h := &rktHandle{
                proc:   proc,
                name:   qpid.Name,
                logger: d.logger,
                doneCh: make(chan struct{}),
                waitCh: make(chan error, 1),
        }

        go h.run()
        return h, nil
}

func (h *rktHandle) ID() string {
        // Return a handle to the PID
        pid := &rktPID{
                Pid:  h.proc.Pid,
                Name: h.name,
        }
        data, err := json.Marshal(pid)
        if err != nil {
                h.logger.Printf("[ERR] failed to marshal rkt PID to JSON: %s", err)
        }
        return fmt.Sprintf("Rkt:%s", string(data))
}

func (h *rktHandle) WaitCh() chan error {
        return h.waitCh
}

func (h *rktHandle) Update(task *structs.Task) error {
        // Update is not possible
        return nil
}

// Kill is used to terminate the task. We send an Interrupt
// and then provide a 5 second grace period before doing a Kill.
func (h *rktHandle) Kill() error {
        h.proc.Signal(os.Interrupt)
        select {
        case <-h.doneCh:
                return nil
        case <-time.After(5 * time.Second):
                return h.proc.Kill()
        }
}

func (h *rktHandle) run() {
        ps, err := h.proc.Wait()
        close(h.doneCh)
        if err != nil {
                h.waitCh <- err
        } else if !ps.Success() {
                h.waitCh <- fmt.Errorf("task exited with error")
        }
        close(h.waitCh)
}
<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// File utilities.
// Author: bratseth

package util

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

// Returns true if the given path exists
func PathExists(path string) bool {
	_, err := os.Stat(path)
	return !errors.Is(err, os.ErrNotExist)
}

// Returns true is the given path points to an existing directory
func IsDirectory(path string) bool {
	info, err := os.Stat(path)
	return !errors.Is(err, os.ErrNotExist) && info.IsDir()
}

// Returns the content of a reader as a string
func ReaderToString(reader io.Reader) string {
	var buffer strings.Builder
	io.Copy(&buffer, reader)
	return buffer.String()
}

// Returns the content of a reader as a byte array
func ReaderToBytes(reader io.Reader) []byte {
	var buffer bytes.Buffer
	buffer.ReadFrom(reader)
	return buffer.Bytes()
}

// Returns the contents of reader as indented JSON
func ReaderToJSON(reader io.Reader) string {
	bodyBytes, _ := ioutil.ReadAll(reader)
	var prettyJSON bytes.Buffer
	parseError := json.Indent(&prettyJSON, bodyBytes, "", "    ")
	if parseError != nil { // Not JSON: Print plainly
		return string(bodyBytes)
	}
	return prettyJSON.String()
}

// AtomicWriteFile atomically writes data to filename.
func AtomicWriteFile(filename string, data []byte) error {
	tmpFile, err := ioutil.TempFile("", "vespa")
	if err != nil {
		return err
	}
	defer os.Remove(tmpFile.Name())
	if _, err := tmpFile.Write(data); err != nil {
		return err
	}
	if err := tmpFile.Close(); err != nil {
		return err
	}
	return os.Rename(tmpFile.Name(), filename)
}
<commit_msg>Write temporary file to same directory as final file<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// File utilities.
// Author: bratseth

package util

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// Returns true if the given path exists
func PathExists(path string) bool {
	_, err := os.Stat(path)
	return !errors.Is(err, os.ErrNotExist)
}

// Returns true is the given path points to an existing directory
func IsDirectory(path string) bool {
	info, err := os.Stat(path)
	return !errors.Is(err, os.ErrNotExist) && info.IsDir()
}

// Returns the content of a reader as a string
func ReaderToString(reader io.Reader) string {
	var buffer strings.Builder
	io.Copy(&buffer, reader)
	return buffer.String()
}

// Returns the content of a reader as a byte array
func ReaderToBytes(reader io.Reader) []byte {
	var buffer bytes.Buffer
	buffer.ReadFrom(reader)
	return buffer.Bytes()
}

// Returns the contents of reader as indented JSON
func ReaderToJSON(reader io.Reader) string {
	bodyBytes, _ := ioutil.ReadAll(reader)
	var prettyJSON bytes.Buffer
	parseError := json.Indent(&prettyJSON, bodyBytes, "", "    ")
	if parseError != nil { // Not JSON: Print plainly
		return string(bodyBytes)
	}
	return prettyJSON.String()
}

// AtomicWriteFile atomically writes data to filename.
func AtomicWriteFile(filename string, data []byte) error {
	dir := filepath.Dir(filename)
	tmpFile, err := ioutil.TempFile(dir, "vespa")
	if err != nil {
		return err
	}
	defer os.Remove(tmpFile.Name())
	if _, err := tmpFile.Write(data); err != nil {
		return err
	}
	if err := tmpFile.Close(); err != nil {
		return err
	}
	return os.Rename(tmpFile.Name(), filename)
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"

	"github.com/fsufitch/prez-tweet/prez-tweet-server/model"
)

// InsertTweet inserts a tweet into the transaction
func InsertTweet(tx *sql.Tx, tw model.DBTweet) error {
	_, err := tx.Exec(`
    INSERT INTO tweets (tweet_id_str, screen_name, created_at, body)
    VALUES ($1, $2, $3, $4);
  `, tw.IDStr, tw.ScreenName, tw.CreatedAt, tw.Body)
	return err
}

// GetTweetsFromIDs recovers any tweets available for the given IDs
func GetTweetsFromIDs(tx *sql.Tx, StrIDs []string) (tweetMap map[string]model.DBTweet, err error) {
	queryArgsStr, iArgs := stringQueryArgsList(StrIDs)

	rows, err := tx.Query(`
    SELECT t.tweet_id_str, t.screen_name, t.created_at, t.body
    FROM tweets t
    WHERE t.tweet_id_str IN (`+queryArgsStr+`)
	`, iArgs...)
	if err != nil {
		return
	}
	defer rows.Close()

	tweetMap = map[string]model.DBTweet{}
	for rows.Next() {
		var t model.DBTweet
		rows.Scan(&t.IDStr, &t.ScreenName, &t.CreatedAt, &t.Body)
		tweetMap[t.IDStr] = t
	}
	return
}

// GetMostRecentTweet recovers the most recent tweet for a given presidential author
func GetMostRecentTweet(tx *sql.Tx, author model.TweetAuthor) (t model.DBTweet, err error) {
	queryArgsStr, iArgs := stringQueryArgsList(author.ScreenNames)
	row := tx.QueryRow(`
		SELECT t.tweet_id_str, t.screen_name, t.created_at, t.body
		FROM tweets t
		WHERE t.screen_name IN (`+queryArgsStr+`)
		ORDER BY t.created_at DESC
		LIMIT 1;
	`, iArgs...)
	err = row.Scan(&t.IDStr, &t.ScreenName, &t.CreatedAt, &t.Body)
	return
}
<commit_msg>Close logic holes<commit_after>package db

import (
	"database/sql"
	"fmt"

	"github.com/fsufitch/prez-tweet/prez-tweet-server/model"
)

// InsertTweet inserts a tweet into the transaction
func InsertTweet(tx *sql.Tx, tw model.DBTweet) error {
	_, err := tx.Exec(`
    INSERT INTO tweets (tweet_id_str, screen_name, created_at, body)
    VALUES ($1, $2, $3, $4);
  `, tw.IDStr, tw.ScreenName, tw.CreatedAt, tw.Body)
	return err
}

// GetTweetsFromIDs recovers any tweets available for the given IDs
func GetTweetsFromIDs(tx *sql.Tx, StrIDs []string) (tweetMap map[string]model.DBTweet, err error) {
	if len(StrIDs) == 0 {
		return map[string]model.DBTweet{}, nil
	}
	queryArgsStr, iArgs := stringQueryArgsList(StrIDs)

	rows, err := tx.Query(`
    SELECT t.tweet_id_str, t.screen_name, t.created_at, t.body
    FROM tweets t
    WHERE t.tweet_id_str IN (`+queryArgsStr+`)
	`, iArgs...)
	if err != nil {
		return
	}
	defer rows.Close()

	tweetMap = map[string]model.DBTweet{}
	for rows.Next() {
		var t model.DBTweet
		rows.Scan(&t.IDStr, &t.ScreenName, &t.CreatedAt, &t.Body)
		tweetMap[t.IDStr] = t
	}
	return
}

// GetMostRecentTweet recovers the most recent tweet for a given presidential author
func GetMostRecentTweet(tx *sql.Tx, author model.TweetAuthor) (t model.DBTweet, err error) {
	if len(author.ScreenNames) == 0 {
		return model.DBTweet{}, fmt.Errorf("Author has no screen names: %v", author)
	}
	queryArgsStr, iArgs := stringQueryArgsList(author.ScreenNames)
	row := tx.QueryRow(`
		SELECT t.tweet_id_str, t.screen_name, t.created_at, t.body
		FROM tweets t
		WHERE t.screen_name IN (`+queryArgsStr+`)
		ORDER BY t.created_at DESC
		LIMIT 1;
	`, iArgs...)
	err = row.Scan(&t.IDStr, &t.ScreenName, &t.CreatedAt, &t.Body)
	return
}
<|endoftext|>


<commit_before>package wallet

import (
	"bytes"
	"crypto/rand"
	"encoding/gob"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/FactomProject/factom"
	"github.com/FactomProject/factomd/common/interfaces"
	"github.com/FactomProject/factomd/common/primitives"
	"github.com/FactomProject/factomd/database/databaseOverlay"
	"github.com/FactomProject/factomd/database/hybridDB"
	"github.com/FactomProject/factomd/database/mapdb"
	"github.com/FactomProject/go-bip32"
	"github.com/FactomProject/go-bip39"
)

// Database keys and key prefixes
var (
	fcDBPrefix = []byte("Factoids")
	ecDBPrefix = []byte("Entry Credits")
	seedDBKey  = []byte("DB Seed")
)

type WalletDatabaseOverlay struct {
	dbo databaseOverlay.Overlay
}

func NewWalletOverlay(db interfaces.IDatabase) *WalletDatabaseOverlay {
	answer := new(WalletDatabaseOverlay)
	answer.dbo.DB = db
	return answer
}

func NewMapDB() *WalletDatabaseOverlay {
	return NewWalletOverlay(new(mapdb.MapDB))
}

func NewLevelDB(ldbpath string) (*WalletDatabaseOverlay, error) {
	db, err := hybridDB.NewLevelMapHybridDB(ldbpath, false)
	if err != nil {
		fmt.Printf("err opening db: %v\n", err)
	}

	if db == nil {
		fmt.Println("Creating new db ...")
		db, err = hybridDB.NewLevelMapHybridDB(ldbpath, true)

		if err != nil {
			return nil, err
		}
	}
	fmt.Println("Database started from: " + ldbpath)
	return NewWalletOverlay(db), nil
}

func NewBoltDB(boltPath string) (*WalletDatabaseOverlay, error) {
	// check if the file exists or if it is a directory
	fileInfo, err := os.Stat(boltPath)
	if err == nil {
		if fileInfo.IsDir() {
			return nil, fmt.Errorf("The path %s is a directory.  Please specify a file name.", boltPath)
		}
	}

	// create the wallet directory if it doesn't already exist
	if os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(boltPath), 0700); err != nil {
			fmt.Printf("database error %s\n", err)
		}
	}

	if err != nil && !os.IsNotExist(err) { //some other error, besides the file not existing
		fmt.Printf("database error %s\n", err)
		return nil, err
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("Could not use wallet file \"%s\"\n%v\n", boltPath, r)
			os.Exit(1)
		}
	}()
	db := hybridDB.NewBoltMapHybridDB(nil, boltPath)

	fmt.Println("Database started from: " + boltPath)
	return NewWalletOverlay(db), nil
}

type DBSeedBase struct {
	MnemonicSeed            string
	NextFactoidAddressIndex uint32
	NextECAddressIndex      uint32
}

type DBSeed struct {
	DBSeedBase
}

var _ interfaces.BinaryMarshallable = (*DBSeed)(nil)

func (e *DBSeed) MarshalBinary() ([]byte, error) {
	var data primitives.Buffer

	enc := gob.NewEncoder(&data)

	err := enc.Encode(e.DBSeedBase)
	if err != nil {
		return nil, err
	}
	return data.DeepCopyBytes(), nil
}

func (e *DBSeed) UnmarshalBinaryData(data []byte) (newData []byte, err error) {
	dec := gob.NewDecoder(primitives.NewBuffer(data))
	dbsb := DBSeedBase{}
	err = dec.Decode(&dbsb)
	if err != nil {
		return nil, err
	}
	e.DBSeedBase = dbsb
	return nil, nil
}

func (e *DBSeed) UnmarshalBinary(data []byte) (err error) {
	_, err = e.UnmarshalBinaryData(data)
	return
}

func (e *DBSeed) JSONByte() ([]byte, error) {
	return primitives.EncodeJSON(e)
}

func (e *DBSeed) JSONString() (string, error) {
	return primitives.EncodeJSONString(e)
}

func (e *DBSeed) JSONBuffer(b *bytes.Buffer) error {
	return primitives.EncodeJSONToBuffer(e, b)
}

func (e *DBSeed) String() string {
	str, _ := e.JSONString()
	return str
}

func (e *DBSeed) NextFCTAddress() (*factom.FactoidAddress, error) {
	add, err := factom.MakeBIP44FactoidAddress(
		e.MnemonicSeed,
		bip32.FirstHardenedChild,
		0,
		e.NextFactoidAddressIndex,
	)
	if err != nil {
		return nil, err
	}
	e.NextFactoidAddressIndex++
	return add, nil
}

func (e *DBSeed) NextECAddress() (*factom.ECAddress, error) {
	add, err := factom.MakeBIP44ECAddress(
		e.MnemonicSeed,
		bip32.FirstHardenedChild,
		0,
		e.NextECAddressIndex,
	)
	if err != nil {
		return nil, err
	}
	e.NextECAddressIndex++
	return add, nil
}

func NewRandomSeed() (*DBSeed, error) {
	seed := make([]byte, 16)
	if n, err := rand.Read(seed); err != nil {
		panic(err)
		return nil, err
	} else if n != 16 {
		return nil, fmt.Errorf("Wrong number of bytes read: %d", n)
	}

	mnemonic, err := bip39.NewMnemonic(seed)
	if err != nil {
		panic(err)
		return nil, err
	}

	dbSeed := new(DBSeed)
	dbSeed.MnemonicSeed = mnemonic

	return dbSeed, nil
}

func (db *WalletDatabaseOverlay) InsertDBSeed(seed *DBSeed) error {
	if seed == nil {
		return nil
	}

	batch := []interfaces.Record{}
	batch = append(batch, interfaces.Record{seedDBKey, seedDBKey, seed})

	return db.dbo.PutInBatch(batch)
}

func (db *WalletDatabaseOverlay) GetDBSeed() (*DBSeed, error) {
	data, err := db.dbo.Get(seedDBKey, seedDBKey, new(DBSeed))
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, nil
	}
	return data.(*DBSeed), nil
}

func (db *WalletDatabaseOverlay) GetOrCreateDBSeed() (*DBSeed, error) {
	data, err := db.dbo.Get(seedDBKey, seedDBKey, new(DBSeed))
	if err != nil {
		return nil, err
	}
	if data == nil {
		seed, err := NewRandomSeed()
		if err != nil {
			return nil, err
		}
		err = db.InsertDBSeed(seed)
		if err != nil {
			return nil, err
		}
		return seed, nil
	}
	return data.(*DBSeed), nil
}

func (db *WalletDatabaseOverlay) GetNextECAddress() (*factom.ECAddress, error) {
	seed, err := db.GetOrCreateDBSeed()
	if err != nil {
		return nil, err
	}
	add, err := seed.NextECAddress()
	if err != nil {
		return nil, err
	}
	err = db.InsertDBSeed(seed)
	if err != nil {
		return nil, err
	}
	err = db.InsertECAddress(add)
	if err != nil {
		return nil, err
	}
	return add, nil
}

func (db *WalletDatabaseOverlay) GetNextFCTAddress() (*factom.FactoidAddress, error) {
	seed, err := db.GetOrCreateDBSeed()
	if err != nil {
		return nil, err
	}
	add, err := seed.NextFCTAddress()
	if err != nil {
		return nil, err
	}
	err = db.InsertDBSeed(seed)
	if err != nil {
		return nil, err
	}
	err = db.InsertFCTAddress(add)
	if err != nil {
		return nil, err
	}
	return add, nil
}

func (db *WalletDatabaseOverlay) InsertECAddress(e *factom.ECAddress) error {
	if e == nil {
		return nil
	}

	batch := []interfaces.Record{}
	batch = append(batch, interfaces.Record{ecDBPrefix, []byte(e.PubString()), e})

	return db.dbo.PutInBatch(batch)
}

func (db *WalletDatabaseOverlay) GetECAddress(pubString string) (*factom.ECAddress, error) {
	data, err := db.dbo.Get(ecDBPrefix, []byte(pubString), new(factom.ECAddress))
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, nil
	}
	return data.(*factom.ECAddress), nil
}

func (db *WalletDatabaseOverlay) GetAllECAddresses() ([]*factom.ECAddress, error) {
	list, err := db.dbo.FetchAllBlocksFromBucket(ecDBPrefix, new(ECA))
	if err != nil {
		return nil, err
	}
	return toECList(list), nil
}

func toECList(source []interfaces.BinaryMarshallableAndCopyable) []*factom.ECAddress {
	answer := make([]*factom.ECAddress, len(source))
	for i, v := range source {
		answer[i] = v.(*ECA).ECAddress
	}
	sort.Sort(byECName(answer))
	return answer
}

type byECName []*factom.ECAddress

func (f byECName) Len() int {
	return len(f)
}
func (f byECName) Less(i, j int) bool {
	a := strings.Compare(f[i].PubString(), f[j].PubString())
	return a < 0
}
func (f byECName) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (db *WalletDatabaseOverlay) InsertFCTAddress(e *factom.FactoidAddress) error {
	if e == nil {
		return nil
	}

	batch := []interfaces.Record{}
	batch = append(batch, interfaces.Record{fcDBPrefix, []byte(e.String()), e})

	return db.dbo.PutInBatch(batch)
}

func (db *WalletDatabaseOverlay) GetFCTAddress(str string) (*factom.FactoidAddress, error) {
	data, err := db.dbo.Get(fcDBPrefix, []byte(str), new(factom.FactoidAddress))
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, ErrNoSuchAddress
	}
	return data.(*factom.FactoidAddress), nil
}

func (db *WalletDatabaseOverlay) GetAllFCTAddresses() ([]*factom.FactoidAddress, error) {
	list, err := db.dbo.FetchAllBlocksFromBucket(fcDBPrefix, new(FA))
	if err != nil {
		return nil, err
	}
	return toFList(list), nil
}

func toFList(source []interfaces.BinaryMarshallableAndCopyable) []*factom.FactoidAddress {
	answer := make([]*factom.FactoidAddress, len(source))
	for i, v := range source {
		answer[i] = v.(*FA).FactoidAddress
	}
	sort.Sort(byFName(answer))
	return answer
}

func (db *WalletDatabaseOverlay) RemoveAddress(pubString string) error {
	if len(pubString) == 0 {
		return nil
	}
	if pubString[:1] == "F" {
		return db.dbo.Delete(fcDBPrefix, []byte(pubString))
	} else if pubString[:1] == "E" {
		return db.dbo.Delete(ecDBPrefix, []byte(pubString))
	} else {
		return fmt.Errorf("Unknown address type")
	}

	return nil
}

type byFName []*factom.FactoidAddress

func (f byFName) Len() int {
	return len(f)
}
func (f byFName) Less(i, j int) bool {
	a := strings.Compare(f[i].String(), f[j].String())
	return a < 0
}
func (f byFName) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

type ECA struct {
	*factom.ECAddress
}

var _ interfaces.BinaryMarshallableAndCopyable = (*ECA)(nil)

func (t *ECA) New() interfaces.BinaryMarshallableAndCopyable {
	e := new(ECA)
	e.ECAddress = factom.NewECAddress()
	return e
}

type FA struct {
	*factom.FactoidAddress
}

var _ interfaces.BinaryMarshallableAndCopyable = (*FA)(nil)

func (t *FA) New() interfaces.BinaryMarshallableAndCopyable {
	e := new(FA)
	e.FactoidAddress = factom.NewFactoidAddress()
	return e
}
<commit_msg>flush the private key<commit_after>package wallet

import (
	"bytes"
	"crypto/rand"
	"encoding/gob"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/FactomProject/factom"
	"github.com/FactomProject/factomd/common/interfaces"
	"github.com/FactomProject/factomd/common/primitives"
	"github.com/FactomProject/factomd/database/databaseOverlay"
	"github.com/FactomProject/factomd/database/hybridDB"
	"github.com/FactomProject/factomd/database/mapdb"
	"github.com/FactomProject/go-bip32"
	"github.com/FactomProject/go-bip39"
)

// Database keys and key prefixes
var (
	fcDBPrefix = []byte("Factoids")
	ecDBPrefix = []byte("Entry Credits")
	seedDBKey  = []byte("DB Seed")
)

type WalletDatabaseOverlay struct {
	dbo databaseOverlay.Overlay
}

func NewWalletOverlay(db interfaces.IDatabase) *WalletDatabaseOverlay {
	answer := new(WalletDatabaseOverlay)
	answer.dbo.DB = db
	return answer
}

func NewMapDB() *WalletDatabaseOverlay {
	return NewWalletOverlay(new(mapdb.MapDB))
}

func NewLevelDB(ldbpath string) (*WalletDatabaseOverlay, error) {
	db, err := hybridDB.NewLevelMapHybridDB(ldbpath, false)
	if err != nil {
		fmt.Printf("err opening db: %v\n", err)
	}

	if db == nil {
		fmt.Println("Creating new db ...")
		db, err = hybridDB.NewLevelMapHybridDB(ldbpath, true)

		if err != nil {
			return nil, err
		}
	}
	fmt.Println("Database started from: " + ldbpath)
	return NewWalletOverlay(db), nil
}

func NewBoltDB(boltPath string) (*WalletDatabaseOverlay, error) {
	// check if the file exists or if it is a directory
	fileInfo, err := os.Stat(boltPath)
	if err == nil {
		if fileInfo.IsDir() {
			return nil, fmt.Errorf("The path %s is a directory.  Please specify a file name.", boltPath)
		}
	}

	// create the wallet directory if it doesn't already exist
	if os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(boltPath), 0700); err != nil {
			fmt.Printf("database error %s\n", err)
		}
	}

	if err != nil && !os.IsNotExist(err) { //some other error, besides the file not existing
		fmt.Printf("database error %s\n", err)
		return nil, err
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("Could not use wallet file \"%s\"\n%v\n", boltPath, r)
			os.Exit(1)
		}
	}()
	db := hybridDB.NewBoltMapHybridDB(nil, boltPath)

	fmt.Println("Database started from: " + boltPath)
	return NewWalletOverlay(db), nil
}

type DBSeedBase struct {
	MnemonicSeed            string
	NextFactoidAddressIndex uint32
	NextECAddressIndex      uint32
}

type DBSeed struct {
	DBSeedBase
}

var _ interfaces.BinaryMarshallable = (*DBSeed)(nil)

func (e *DBSeed) MarshalBinary() ([]byte, error) {
	var data primitives.Buffer

	enc := gob.NewEncoder(&data)

	err := enc.Encode(e.DBSeedBase)
	if err != nil {
		return nil, err
	}
	return data.DeepCopyBytes(), nil
}

func (e *DBSeed) UnmarshalBinaryData(data []byte) (newData []byte, err error) {
	dec := gob.NewDecoder(primitives.NewBuffer(data))
	dbsb := DBSeedBase{}
	err = dec.Decode(&dbsb)
	if err != nil {
		return nil, err
	}
	e.DBSeedBase = dbsb
	return nil, nil
}

func (e *DBSeed) UnmarshalBinary(data []byte) (err error) {
	_, err = e.UnmarshalBinaryData(data)
	return
}

func (e *DBSeed) JSONByte() ([]byte, error) {
	return primitives.EncodeJSON(e)
}

func (e *DBSeed) JSONString() (string, error) {
	return primitives.EncodeJSONString(e)
}

func (e *DBSeed) JSONBuffer(b *bytes.Buffer) error {
	return primitives.EncodeJSONToBuffer(e, b)
}

func (e *DBSeed) String() string {
	str, _ := e.JSONString()
	return str
}

func (e *DBSeed) NextFCTAddress() (*factom.FactoidAddress, error) {
	add, err := factom.MakeBIP44FactoidAddress(
		e.MnemonicSeed,
		bip32.FirstHardenedChild,
		0,
		e.NextFactoidAddressIndex,
	)
	if err != nil {
		return nil, err
	}
	e.NextFactoidAddressIndex++
	return add, nil
}

func (e *DBSeed) NextECAddress() (*factom.ECAddress, error) {
	add, err := factom.MakeBIP44ECAddress(
		e.MnemonicSeed,
		bip32.FirstHardenedChild,
		0,
		e.NextECAddressIndex,
	)
	if err != nil {
		return nil, err
	}
	e.NextECAddressIndex++
	return add, nil
}

func NewRandomSeed() (*DBSeed, error) {
	seed := make([]byte, 16)
	if n, err := rand.Read(seed); err != nil {
		panic(err)
		return nil, err
	} else if n != 16 {
		return nil, fmt.Errorf("Wrong number of bytes read: %d", n)
	}

	mnemonic, err := bip39.NewMnemonic(seed)
	if err != nil {
		panic(err)
		return nil, err
	}

	dbSeed := new(DBSeed)
	dbSeed.MnemonicSeed = mnemonic

	return dbSeed, nil
}

func (db *WalletDatabaseOverlay) InsertDBSeed(seed *DBSeed) error {
	if seed == nil {
		return nil
	}

	batch := []interfaces.Record{}
	batch = append(batch, interfaces.Record{seedDBKey, seedDBKey, seed})

	return db.dbo.PutInBatch(batch)
}

func (db *WalletDatabaseOverlay) GetDBSeed() (*DBSeed, error) {
	data, err := db.dbo.Get(seedDBKey, seedDBKey, new(DBSeed))
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, nil
	}
	return data.(*DBSeed), nil
}

func (db *WalletDatabaseOverlay) GetOrCreateDBSeed() (*DBSeed, error) {
	data, err := db.dbo.Get(seedDBKey, seedDBKey, new(DBSeed))
	if err != nil {
		return nil, err
	}
	if data == nil {
		seed, err := NewRandomSeed()
		if err != nil {
			return nil, err
		}
		err = db.InsertDBSeed(seed)
		if err != nil {
			return nil, err
		}
		return seed, nil
	}
	return data.(*DBSeed), nil
}

func (db *WalletDatabaseOverlay) GetNextECAddress() (*factom.ECAddress, error) {
	seed, err := db.GetOrCreateDBSeed()
	if err != nil {
		return nil, err
	}
	add, err := seed.NextECAddress()
	if err != nil {
		return nil, err
	}
	err = db.InsertDBSeed(seed)
	if err != nil {
		return nil, err
	}
	err = db.InsertECAddress(add)
	if err != nil {
		return nil, err
	}
	return add, nil
}

func (db *WalletDatabaseOverlay) GetNextFCTAddress() (*factom.FactoidAddress, error) {
	seed, err := db.GetOrCreateDBSeed()
	if err != nil {
		return nil, err
	}
	add, err := seed.NextFCTAddress()
	if err != nil {
		return nil, err
	}
	err = db.InsertDBSeed(seed)
	if err != nil {
		return nil, err
	}
	err = db.InsertFCTAddress(add)
	if err != nil {
		return nil, err
	}
	return add, nil
}

func (db *WalletDatabaseOverlay) InsertECAddress(e *factom.ECAddress) error {
	if e == nil {
		return nil
	}

	batch := []interfaces.Record{}
	batch = append(batch, interfaces.Record{ecDBPrefix, []byte(e.PubString()), e})

	return db.dbo.PutInBatch(batch)
}

func (db *WalletDatabaseOverlay) GetECAddress(pubString string) (*factom.ECAddress, error) {
	data, err := db.dbo.Get(ecDBPrefix, []byte(pubString), new(factom.ECAddress))
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, nil
	}
	return data.(*factom.ECAddress), nil
}

func (db *WalletDatabaseOverlay) GetAllECAddresses() ([]*factom.ECAddress, error) {
	list, err := db.dbo.FetchAllBlocksFromBucket(ecDBPrefix, new(ECA))
	if err != nil {
		return nil, err
	}
	return toECList(list), nil
}

func toECList(source []interfaces.BinaryMarshallableAndCopyable) []*factom.ECAddress {
	answer := make([]*factom.ECAddress, len(source))
	for i, v := range source {
		answer[i] = v.(*ECA).ECAddress
	}
	sort.Sort(byECName(answer))
	return answer
}

type byECName []*factom.ECAddress

func (f byECName) Len() int {
	return len(f)
}
func (f byECName) Less(i, j int) bool {
	a := strings.Compare(f[i].PubString(), f[j].PubString())
	return a < 0
}
func (f byECName) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (db *WalletDatabaseOverlay) InsertFCTAddress(e *factom.FactoidAddress) error {
	if e == nil {
		return nil
	}

	batch := []interfaces.Record{}
	batch = append(batch, interfaces.Record{fcDBPrefix, []byte(e.String()), e})

	return db.dbo.PutInBatch(batch)
}

func (db *WalletDatabaseOverlay) GetFCTAddress(str string) (*factom.FactoidAddress, error) {
	data, err := db.dbo.Get(fcDBPrefix, []byte(str), new(factom.FactoidAddress))
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, ErrNoSuchAddress
	}
	return data.(*factom.FactoidAddress), nil
}

func (db *WalletDatabaseOverlay) GetAllFCTAddresses() ([]*factom.FactoidAddress, error) {
	list, err := db.dbo.FetchAllBlocksFromBucket(fcDBPrefix, new(FA))
	if err != nil {
		return nil, err
	}
	return toFList(list), nil
}

func toFList(source []interfaces.BinaryMarshallableAndCopyable) []*factom.FactoidAddress {
	answer := make([]*factom.FactoidAddress, len(source))
	for i, v := range source {
		answer[i] = v.(*FA).FactoidAddress
	}
	sort.Sort(byFName(answer))
	return answer
}

func (db *WalletDatabaseOverlay) RemoveAddress(pubString string) error {
	if len(pubString) == 0 {
		return nil
	}
	if pubString[:1] == "F" {
		err := db.dbo.Delete(fcDBPrefix, []byte(pubString))
		if err == nil {
			err := db.dbo.Delete(fcDBPrefix, []byte(pubString)) //delete twice to flush the db file
			return err
		} else { return err }
	} else if pubString[:1] == "E" {
		err := db.dbo.Delete(ecDBPrefix, []byte(pubString))
		if err == nil {
			err := db.dbo.Delete(ecDBPrefix, []byte(pubString)) //delete twice to flush the db file
			return err
		} else { return err }
	} else {
		return fmt.Errorf("Unknown address type")
	}

	return nil
}

type byFName []*factom.FactoidAddress

func (f byFName) Len() int {
	return len(f)
}
func (f byFName) Less(i, j int) bool {
	a := strings.Compare(f[i].String(), f[j].String())
	return a < 0
}
func (f byFName) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

type ECA struct {
	*factom.ECAddress
}

var _ interfaces.BinaryMarshallableAndCopyable = (*ECA)(nil)

func (t *ECA) New() interfaces.BinaryMarshallableAndCopyable {
	e := new(ECA)
	e.ECAddress = factom.NewECAddress()
	return e
}

type FA struct {
	*factom.FactoidAddress
}

var _ interfaces.BinaryMarshallableAndCopyable = (*FA)(nil)

func (t *FA) New() interfaces.BinaryMarshallableAndCopyable {
	e := new(FA)
	e.FactoidAddress = factom.NewFactoidAddress()
	return e
}
<|endoftext|>


<commit_before>// Binary gcmap produces gcmap.com URL for your historical and upcoming travel.
package main

import (
	"github.com/mrjones/oauth"
	"github.com/seanrees/tripist/config"
	"github.com/seanrees/tripist/tripit"
	"log"
	"os"
	"sort"
	"strings"
)

type byStartDate []tripit.Segment

func (a byStartDate) Len() int      { return len(a) }
func (a byStartDate) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byStartDate) Less(i, j int) bool {
	s, err := a[i].StartDateTime.Parse()
	if err != nil {
		return true
	}
	e, err := a[j].StartDateTime.Parse()
	if err != nil {
		return false
	}
	return s.Before(e)
}

type airportCounter map[string]int
type airportCounterSort struct {
	ac   airportCounter
	keys []string
}

func (a airportCounter) increment(code string) {
	a[code] = a[code] + 1
}

// Returns sorted keys (descending).
func (a airportCounter) sortedKeys() []string {
	var keys []string

	for k := range a {
		keys = append(keys, k)
	}

	s := airportCounterSort{a, keys}
	sort.Sort(s)

	return s.keys
}

func (a airportCounterSort) Len() int      { return len(a.keys) }
func (a airportCounterSort) Swap(i, j int) { a.keys[i], a.keys[j] = a.keys[j], a.keys[i] }
func (a airportCounterSort) Less(i, j int) bool {
	return a.ac[a.keys[i]] > a.ac[a.keys[j]]
}

func main() {
	const configFilename = "user.json"
	const tripistConfigFilename = "tripist.json"

	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	log.SetOutput(os.Stderr)

	var conf config.UserKeys
	if _, err := os.Stat(configFilename); err == nil {
		conf, err = config.Read(configFilename)
		if err != nil {
			log.Fatalf("Unable to read configuration: %v\n", err)
		}
	} else {
		log.Fatalf("Configuration file %q does not exist.\n", configFilename)
	}

	if err := config.LoadAPIKeys(tripistConfigFilename); err != nil {
		log.Println("Using built-in keys.")
	} else {
		log.Printf("Loaded API keys from %s", tripistConfigFilename)
	}

	token := &oauth.AccessToken{
		Token:  conf.TripitToken,
		Secret: conf.TripitSecret,
	}

	api := tripit.NewTripitV1API(token)
	lp := tripit.ListParameters{
		Traveler:       "true",
		Past:           true,
		ModifiedSince:  0,
		IncludeObjects: true,
	}

	var page int64 = 1
	var segmentsByTrip [][]tripit.Segment

	for {
		lp.PageNum = page
		tr, err := api.ListRaw(&lp)

		if err != nil {
			log.Printf("Got error: %v\n", err)
		}

		log.Printf("Loaded %d (page=%d) trips\n", len(tr.Trip), page)
		for i, a := range tr.Trip {
			log.Printf("Trip %3d: %s\n", (page-1)*tr.PageSize+int64(i), a.DisplayName)
		}

		for _, trip := range tr.Trip {
			var segs []tripit.Segment
			for _, ao := range tr.AirObject {
				if ao.TripId == trip.Id {
					segs = append(segs, ao.Segments()...)
				}
			}
			segmentsByTrip = append(segmentsByTrip, segs)
		}

		page += 1
		if page > tr.MaxPage {
			break
		}
	}

	var paths []string
	airports := make(airportCounter)

	for _, segs := range segmentsByTrip {
		sort.Sort(byStartDate(segs))
		last := ""
		path := ""
		for _, s := range segs {
			if last != s.StartAirportCode {
				if len(path) > 0 {
					paths = append(paths, path)
				}
				path = s.StartAirportCode
				airports.increment(s.StartAirportCode)
			}
			path += "-" + s.EndAirportCode
			last = s.EndAirportCode

			airports.increment(s.EndAirportCode)
		}
		if len(path) > 0 {
			paths = append(paths, path)
		}
	}

	log.Printf("Top airports:\n")
	for i, k := range airports.sortedKeys() {
		log.Printf("  #%02d: %s (%d visits)\n", i, k, airports[k])
	}

	log.Printf("http://www.gcmap.com/mapui?P=%s", strings.Join(paths, ","))
}
<commit_msg>Add year range filter: -start_year and -end_year<commit_after>// Binary gcmap produces gcmap.com URL for your historical and upcoming travel.
package main

import (
	"flag"
	"github.com/mrjones/oauth"
	"github.com/seanrees/tripist/config"
	"github.com/seanrees/tripist/tripit"
	"log"
	"os"
	"sort"
	"strings"
)

var (
	startYear = flag.Int("start_year", 0, "Only consider trips after (incusive) of start_year.")
	endYear   = flag.Int("end_year", 9999, "Only consider trips before (inclusive) of end_year.")
)

type byStartDate []tripit.Segment

func (a byStartDate) Len() int      { return len(a) }
func (a byStartDate) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byStartDate) Less(i, j int) bool {
	s, err := a[i].StartDateTime.Parse()
	if err != nil {
		return true
	}
	e, err := a[j].StartDateTime.Parse()
	if err != nil {
		return false
	}
	return s.Before(e)
}

type airportCounter map[string]int
type airportCounterSort struct {
	ac   airportCounter
	keys []string
}

func (a airportCounter) increment(code string) {
	a[code] = a[code] + 1
}

// Returns sorted keys (descending).
func (a airportCounter) sortedKeys() []string {
	var keys []string

	for k := range a {
		keys = append(keys, k)
	}

	s := airportCounterSort{a, keys}
	sort.Sort(s)

	return s.keys
}

func (a airportCounterSort) Len() int      { return len(a.keys) }
func (a airportCounterSort) Swap(i, j int) { a.keys[i], a.keys[j] = a.keys[j], a.keys[i] }
func (a airportCounterSort) Less(i, j int) bool {
	return a.ac[a.keys[i]] > a.ac[a.keys[j]]
}

func main() {
	const configFilename = "user.json"
	const tripistConfigFilename = "tripist.json"

	flag.Parse()

	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	log.SetOutput(os.Stderr)

	var conf config.UserKeys
	if _, err := os.Stat(configFilename); err == nil {
		conf, err = config.Read(configFilename)
		if err != nil {
			log.Fatalf("Unable to read configuration: %v\n", err)
		}
	} else {
		log.Fatalf("Configuration file %q does not exist.\n", configFilename)
	}

	if err := config.LoadAPIKeys(tripistConfigFilename); err != nil {
		log.Println("Using built-in keys.")
	} else {
		log.Printf("Loaded API keys from %s", tripistConfigFilename)
	}

	token := &oauth.AccessToken{
		Token:  conf.TripitToken,
		Secret: conf.TripitSecret,
	}

	api := tripit.NewTripitV1API(token)
	lp := tripit.ListParameters{
		Traveler:       "true",
		Past:           true,
		ModifiedSince:  0,
		IncludeObjects: true,
	}

	var page int64 = 1
	var segmentsByTrip [][]tripit.Segment

	for {
		lp.PageNum = page
		tr, err := api.ListRaw(&lp)

		if err != nil {
			log.Printf("Got error: %v\n", err)
		}

		log.Printf("Loaded %d (page=%d) trips\n", len(tr.Trip), page)
		for i, a := range tr.Trip {
			log.Printf("Trip %3d: %s\n", (page-1)*tr.PageSize+int64(i), a.DisplayName)
		}

		for _, trip := range tr.Trip {
			if sy, ey := trip.ActualStartDate.Year(), trip.ActualEndDate.Year(); sy < *startYear || ey > *endYear {
				log.Printf("Ignoring %q because starts/ends in %d/%d, want in [%d-%d]", trip.DisplayName, sy, ey, *startYear, *endYear)
				continue
			}

			var segs []tripit.Segment
			for _, ao := range tr.AirObject {
				if ao.TripId == trip.Id {
					segs = append(segs, ao.Segments()...)
				}
			}
			segmentsByTrip = append(segmentsByTrip, segs)
		}

		page += 1
		if page > tr.MaxPage {
			break
		}
	}

	var paths []string
	airports := make(airportCounter)

	for _, segs := range segmentsByTrip {
		sort.Sort(byStartDate(segs))
		last := ""
		path := ""
		for _, s := range segs {
			if last != s.StartAirportCode {
				if len(path) > 0 {
					paths = append(paths, path)
				}
				path = s.StartAirportCode
				airports.increment(s.StartAirportCode)
			}
			path += "-" + s.EndAirportCode
			last = s.EndAirportCode

			airports.increment(s.EndAirportCode)
		}
		if len(path) > 0 {
			paths = append(paths, path)
		}
	}

	log.Printf("Top airports:\n")
	for i, k := range airports.sortedKeys() {
		log.Printf("  #%02d: %s (%d visits)\n", i, k, airports[k])
	}

	log.Printf("http://www.gcmap.com/mapui?P=%s", strings.Join(paths, ","))
}
<|endoftext|>


<commit_before>// Copyright © 2021 Vasily Ovchinnikov <vasily@remerge.io>.
//
// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp
// licensed under Apache License 2.0.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsfs

import (
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"syscall"

	"github.com/googleapis/google-cloud-go-testing/storage/stiface"

	"cloud.google.com/go/storage"

	"google.golang.org/api/iterator"
)

// GcsFs is the Afero version adapted for GCS
type GcsFile struct {
	openFlags int
	fhOffset  int64 //File handle specific offset
	closed    bool
	ReadDirIt stiface.ObjectIterator
	resource  *gcsFileResource
}

func NewGcsFile(
	ctx context.Context,
	fs *Fs,
	obj stiface.ObjectHandle,
	openFlags int,
	// Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it
	fileMode os.FileMode,
	name string,
) *GcsFile {
	return &GcsFile{
		openFlags: openFlags,
		fhOffset:  0,
		closed:    false,
		ReadDirIt: nil,
		resource: &gcsFileResource{
			ctx: ctx,
			fs:  fs,

			obj:      obj,
			name:     name,
			fileMode: fileMode,

			currentGcsSize: 0,

			offset: 0,
			reader: nil,
			writer: nil,
		},
	}
}

func NewGcsFileFromOldFH(
	openFlags int,
	fileMode os.FileMode,
	oldFile *gcsFileResource,
) *GcsFile {
	res := &GcsFile{
		openFlags: openFlags,
		fhOffset:  0,
		closed:    false,
		ReadDirIt: nil,

		resource: oldFile,
	}
	res.resource.fileMode = fileMode

	return res
}

func (o *GcsFile) Close() error {
	if o.closed {
		// the afero spec expects the call to Close on a closed file to return an error
		return ErrFileClosed
	}
	o.closed = true
	return o.resource.Close()
}

func (o *GcsFile) Seek(newOffset int64, whence int) (int64, error) {
	if o.closed {
		return 0, ErrFileClosed
	}

	//Since this is an expensive operation; let's make sure we need it
	if (whence == 0 && newOffset == o.fhOffset) || (whence == 1 && newOffset == 0) {
		return o.fhOffset, nil
	}
	log.Printf("WARNING: Seek beavhior triggered, highly inefficent. Offset before seek is at %d\n", o.fhOffset)

	//Fore the reader/writers to be reopened (at correct offset)
	err := o.Sync()
	if err != nil {
		return 0, err
	}
	stat, err := o.Stat()
	if err != nil {
		return 0, nil
	}

	switch whence {
	case 0:
		o.fhOffset = newOffset
	case 1:
		o.fhOffset += newOffset
	case 2:
		o.fhOffset = stat.Size() + newOffset
	}
	return o.fhOffset, nil
}

func (o *GcsFile) Read(p []byte) (n int, err error) {
	return o.ReadAt(p, o.fhOffset)
}

func (o *GcsFile) ReadAt(p []byte, off int64) (n int, err error) {
	if o.closed {
		return 0, ErrFileClosed
	}

	read, err := o.resource.ReadAt(p, off)
	o.fhOffset += int64(read)
	return read, err
}

func (o *GcsFile) Write(p []byte) (n int, err error) {
	return o.WriteAt(p, o.fhOffset)
}

func (o *GcsFile) WriteAt(b []byte, off int64) (n int, err error) {
	if o.closed {
		return 0, ErrFileClosed
	}

	if o.openFlags&os.O_RDONLY != 0 {
		return 0, fmt.Errorf("file is opend as read only")
	}

	_, err = o.resource.obj.Attrs(o.resource.ctx)
	if err != nil {
		if err == storage.ErrObjectNotExist {
			if o.openFlags&os.O_CREATE == 0 {
				return 0, ErrFileNotFound
			}
		} else {
			return 0, fmt.Errorf("error getting file attributes: %v", err)
		}
	}

	written, err := o.resource.WriteAt(b, off)
	o.fhOffset += int64(written)
	return written, err
}

func (o *GcsFile) Name() string {
	return filepath.FromSlash(o.resource.name)
}

func (o *GcsFile) readdirImpl(count int) ([]*FileInfo, error) {
	err := o.Sync()
	if err != nil {
		return nil, err
	}

	var ownInfo os.FileInfo
	ownInfo, err = o.Stat()
	if err != nil {
		return nil, err
	}

	if !ownInfo.IsDir() {
		return nil, syscall.ENOTDIR
	}

	path := o.resource.fs.ensureTrailingSeparator(o.resource.name)
	if o.ReadDirIt == nil {
		//log.Printf("Querying path : %s\n", path)
		bucketName, bucketPath := o.resource.fs.splitName(path)

		o.ReadDirIt = o.resource.fs.client.Bucket(bucketName).Objects(
			o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: bucketPath, Versions: false})
	}
	var res []*FileInfo
	for {
		object, err := o.ReadDirIt.Next()
		if err == iterator.Done {
			// reset the iterator
			o.ReadDirIt = nil

			if len(res) > 0 || count <= 0 {
				return res, nil
			}

			return res, io.EOF
		}
		if err != nil {
			return res, err
		}

		tmp := newFileInfoFromAttrs(object, o.resource.fileMode)

		if tmp.Name() == "" {
			// neither object.Name, not object.Prefix were present - so let's skip this unknown thing
			continue
		}

		if object.Name == "" && object.Prefix == "" {
			continue
		}

		if tmp.Name() == ownInfo.Name() {
			// Hmmm
			continue
		}

		res = append(res, tmp)

		// This would interrupt the iteration, once we reach the count.
		// But it would then have files coming before folders - that's not what we want to have exactly,
		// since it makes the results unpredictable. Hence, we iterate all the objects and then do
		// the cut-off in a higher level method
		//if count > 0 && len(res) >= count {
		//	break
		//}
	}
	//return res, nil
}

func (o *GcsFile) Readdir(count int) ([]os.FileInfo, error) {
	fi, err := o.readdirImpl(count)
	if len(fi) > 0 {
		sort.Sort(ByName(fi))
	}

	if count > 0 {
		fi = fi[:count]
	}

	var res []os.FileInfo
	for _, f := range fi {
		res = append(res, f)
	}
	return res, err
}

func (o *GcsFile) Readdirnames(n int) ([]string, error) {
	fi, err := o.Readdir(n)
	if err != nil && err != io.EOF {
		return nil, err
	}
	names := make([]string, len(fi))

	for i, f := range fi {
		names[i] = f.Name()
	}
	return names, err
}

func (o *GcsFile) Stat() (os.FileInfo, error) {
	err := o.Sync()
	if err != nil {
		return nil, err
	}

	return newFileInfo(o.resource.name, o.resource.fs, o.resource.fileMode)
}

func (o *GcsFile) Sync() error {
	return o.resource.maybeCloseIo()
}

func (o *GcsFile) Truncate(wantedSize int64) error {
	if o.closed {
		return ErrFileClosed
	}
	if o.openFlags == os.O_RDONLY {
		return fmt.Errorf("file was opened as read only")
	}
	return o.resource.Truncate(wantedSize)
}

func (o *GcsFile) WriteString(s string) (ret int, err error) {
	return o.Write([]byte(s))
}
<commit_msg>fix typo<commit_after>// Copyright © 2021 Vasily Ovchinnikov <vasily@remerge.io>.
//
// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp
// licensed under Apache License 2.0.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsfs

import (
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"syscall"

	"github.com/googleapis/google-cloud-go-testing/storage/stiface"

	"cloud.google.com/go/storage"

	"google.golang.org/api/iterator"
)

// GcsFs is the Afero version adapted for GCS
type GcsFile struct {
	openFlags int
	fhOffset  int64 //File handle specific offset
	closed    bool
	ReadDirIt stiface.ObjectIterator
	resource  *gcsFileResource
}

func NewGcsFile(
	ctx context.Context,
	fs *Fs,
	obj stiface.ObjectHandle,
	openFlags int,
	// Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it
	fileMode os.FileMode,
	name string,
) *GcsFile {
	return &GcsFile{
		openFlags: openFlags,
		fhOffset:  0,
		closed:    false,
		ReadDirIt: nil,
		resource: &gcsFileResource{
			ctx: ctx,
			fs:  fs,

			obj:      obj,
			name:     name,
			fileMode: fileMode,

			currentGcsSize: 0,

			offset: 0,
			reader: nil,
			writer: nil,
		},
	}
}

func NewGcsFileFromOldFH(
	openFlags int,
	fileMode os.FileMode,
	oldFile *gcsFileResource,
) *GcsFile {
	res := &GcsFile{
		openFlags: openFlags,
		fhOffset:  0,
		closed:    false,
		ReadDirIt: nil,

		resource: oldFile,
	}
	res.resource.fileMode = fileMode

	return res
}

func (o *GcsFile) Close() error {
	if o.closed {
		// the afero spec expects the call to Close on a closed file to return an error
		return ErrFileClosed
	}
	o.closed = true
	return o.resource.Close()
}

func (o *GcsFile) Seek(newOffset int64, whence int) (int64, error) {
	if o.closed {
		return 0, ErrFileClosed
	}

	//Since this is an expensive operation; let's make sure we need it
	if (whence == 0 && newOffset == o.fhOffset) || (whence == 1 && newOffset == 0) {
		return o.fhOffset, nil
	}
	log.Printf("WARNING: Seek behavior triggered, highly inefficent. Offset before seek is at %d\n", o.fhOffset)

	//Fore the reader/writers to be reopened (at correct offset)
	err := o.Sync()
	if err != nil {
		return 0, err
	}
	stat, err := o.Stat()
	if err != nil {
		return 0, nil
	}

	switch whence {
	case 0:
		o.fhOffset = newOffset
	case 1:
		o.fhOffset += newOffset
	case 2:
		o.fhOffset = stat.Size() + newOffset
	}
	return o.fhOffset, nil
}

func (o *GcsFile) Read(p []byte) (n int, err error) {
	return o.ReadAt(p, o.fhOffset)
}

func (o *GcsFile) ReadAt(p []byte, off int64) (n int, err error) {
	if o.closed {
		return 0, ErrFileClosed
	}

	read, err := o.resource.ReadAt(p, off)
	o.fhOffset += int64(read)
	return read, err
}

func (o *GcsFile) Write(p []byte) (n int, err error) {
	return o.WriteAt(p, o.fhOffset)
}

func (o *GcsFile) WriteAt(b []byte, off int64) (n int, err error) {
	if o.closed {
		return 0, ErrFileClosed
	}

	if o.openFlags&os.O_RDONLY != 0 {
		return 0, fmt.Errorf("file is opend as read only")
	}

	_, err = o.resource.obj.Attrs(o.resource.ctx)
	if err != nil {
		if err == storage.ErrObjectNotExist {
			if o.openFlags&os.O_CREATE == 0 {
				return 0, ErrFileNotFound
			}
		} else {
			return 0, fmt.Errorf("error getting file attributes: %v", err)
		}
	}

	written, err := o.resource.WriteAt(b, off)
	o.fhOffset += int64(written)
	return written, err
}

func (o *GcsFile) Name() string {
	return filepath.FromSlash(o.resource.name)
}

func (o *GcsFile) readdirImpl(count int) ([]*FileInfo, error) {
	err := o.Sync()
	if err != nil {
		return nil, err
	}

	var ownInfo os.FileInfo
	ownInfo, err = o.Stat()
	if err != nil {
		return nil, err
	}

	if !ownInfo.IsDir() {
		return nil, syscall.ENOTDIR
	}

	path := o.resource.fs.ensureTrailingSeparator(o.resource.name)
	if o.ReadDirIt == nil {
		//log.Printf("Querying path : %s\n", path)
		bucketName, bucketPath := o.resource.fs.splitName(path)

		o.ReadDirIt = o.resource.fs.client.Bucket(bucketName).Objects(
			o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: bucketPath, Versions: false})
	}
	var res []*FileInfo
	for {
		object, err := o.ReadDirIt.Next()
		if err == iterator.Done {
			// reset the iterator
			o.ReadDirIt = nil

			if len(res) > 0 || count <= 0 {
				return res, nil
			}

			return res, io.EOF
		}
		if err != nil {
			return res, err
		}

		tmp := newFileInfoFromAttrs(object, o.resource.fileMode)

		if tmp.Name() == "" {
			// neither object.Name, not object.Prefix were present - so let's skip this unknown thing
			continue
		}

		if object.Name == "" && object.Prefix == "" {
			continue
		}

		if tmp.Name() == ownInfo.Name() {
			// Hmmm
			continue
		}

		res = append(res, tmp)

		// This would interrupt the iteration, once we reach the count.
		// But it would then have files coming before folders - that's not what we want to have exactly,
		// since it makes the results unpredictable. Hence, we iterate all the objects and then do
		// the cut-off in a higher level method
		//if count > 0 && len(res) >= count {
		//	break
		//}
	}
	//return res, nil
}

func (o *GcsFile) Readdir(count int) ([]os.FileInfo, error) {
	fi, err := o.readdirImpl(count)
	if len(fi) > 0 {
		sort.Sort(ByName(fi))
	}

	if count > 0 {
		fi = fi[:count]
	}

	var res []os.FileInfo
	for _, f := range fi {
		res = append(res, f)
	}
	return res, err
}

func (o *GcsFile) Readdirnames(n int) ([]string, error) {
	fi, err := o.Readdir(n)
	if err != nil && err != io.EOF {
		return nil, err
	}
	names := make([]string, len(fi))

	for i, f := range fi {
		names[i] = f.Name()
	}
	return names, err
}

func (o *GcsFile) Stat() (os.FileInfo, error) {
	err := o.Sync()
	if err != nil {
		return nil, err
	}

	return newFileInfo(o.resource.name, o.resource.fs, o.resource.fileMode)
}

func (o *GcsFile) Sync() error {
	return o.resource.maybeCloseIo()
}

func (o *GcsFile) Truncate(wantedSize int64) error {
	if o.closed {
		return ErrFileClosed
	}
	if o.openFlags == os.O_RDONLY {
		return fmt.Errorf("file was opened as read only")
	}
	return o.resource.Truncate(wantedSize)
}

func (o *GcsFile) WriteString(s string) (ret int, err error) {
	return o.Write([]byte(s))
}
<|endoftext|>


<commit_before>package generatorControllers

import (
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"eaciit/wfdemo-git/web/helper"
	"time"

	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	"github.com/eaciit/toolkit"
)

type GenDataPeriod struct {
	*BaseController
}

func (d *GenDataPeriod) Generate(base *BaseController) {
	d.BaseController = base
	// conn, e := PrepareConnection()
	// if e != nil {
	// 	toolkit.Println("Scada Summary : " + e.Error())
	// 	os.Exit(0)
	// }
	// defer conn.Close()
	var e error
	conn := d.BaseController.Ctx.Connection

	// reset data
	// #faisal
	// remove this function
	// d.BaseController.Ctx.DeleteMany(NewLatestDataPeriod(), dbox.Ne("projectname", ""))

	projects, _ := helper.GetProjectList()

	for _, proj := range projects {
		projectName := proj.Value
		scadaResults := make([]time.Time, 2)
		dgrResults := make([]time.Time, 2)
		alarmResults := make([]time.Time, 2)
		jmrResults := make([]time.Time, 2)
		metResults := make([]time.Time, 2)
		durationResults := make([]time.Time, 2)
		scadaHFDResult := make([]time.Time, 2)
		warningResult := make([]time.Time, 2)
		eventDownResult := make([]time.Time, 2)
		eventRawResult := make([]time.Time, 2)
		eventDownHFDResult := make([]time.Time, 2)
		scadaOEMResult := make([]time.Time, 2)
		// scadaAnomalyresults := make([]time.Time, 2)
		// alarmOverlappingresults := make([]time.Time, 2)
		// alarmScadaAnomalyresults := make([]time.Time, 2)

		scadaResults[0], scadaResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		dgrResults[0], dgrResults[1], e = getDataDateAvailable(conn, new(DGRModel).TableName(), "dateinfo.dateid", dbox.Eq("site", projectName))
		alarmResults[0], alarmResults[1], e = getDataDateAvailable(conn, new(Alarm).TableName(), "startdate", dbox.Eq("farm", projectName))
		jmrResults[0], jmrResults[1], e = getDataDateAvailable(conn, new(JMR).TableName(), "dateinfo.dateid", nil)
		metResults[0], metResults[1], e = getDataDateAvailable(conn, new(MetTower).TableName(), "timestamp", nil)
		durationResults[0], durationResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", false), dbox.Eq("projectname", projectName)))
		scadaHFDResult[0], scadaHFDResult[1], e = getDataDateAvailable(conn, new(ScadaDataHFD).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		warningResult[0], warningResult[1], e = getDataDateAvailable(conn, new(EventAlarm).TableName(), "timestart", dbox.Eq("projectname", projectName))
		eventDownResult[0], eventDownResult[1], e = getDataDateAvailable(conn, new(EventDown).TableName(), "timestart", dbox.Eq("projectname", projectName))
		eventRawResult[0], eventRawResult[1], e = getDataDateAvailable(conn, new(EventRaw).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		eventDownHFDResult[0], eventDownHFDResult[1], e = getDataDateAvailable(conn, new(EventDownHFD).TableName(), "timestart", dbox.Eq("projectname", projectName))
		scadaOEMResult[0], scadaOEMResult[1], e = getDataDateAvailable(conn, new(ScadaDataOEM).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		// alarmOverlappingresults[0], alarmOverlappingresults[1], e = getDataDateAvailable(conn, new(AlarmOverlapping).TableName(), "startdate", dbox.Eq("farm", projectName))
		// alarmScadaAnomalyresults[0], alarmScadaAnomalyresults[1], e = getDataDateAvailable(conn, new(AlarmScadaAnomaly).TableName(), "startdate", dbox.Eq("farm", projectName))
		// scadaAnomalyresults[0], scadaAnomalyresults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", true), dbox.Eq("projectname", projectName)))
		_ = e
		availdatedata := struct {
			ScadaData    []time.Time
			DGRData      []time.Time
			Alarm        []time.Time
			JMR          []time.Time
			MET          []time.Time
			Duration     []time.Time
			ScadaDataHFD []time.Time
			Warning      []time.Time
			ScadaDataOEM []time.Time
			EventDown    []time.Time
			EventRaw     []time.Time
			EventDownHFD []time.Time
			// ScadaAnomaly      []time.Time
			// AlarmOverlapping  []time.Time
			// AlarmScadaAnomaly []time.Time
		}{
			ScadaData:    scadaResults,
			DGRData:      dgrResults,
			Alarm:        alarmResults,
			JMR:          jmrResults,
			MET:          metResults,
			Duration:     durationResults,
			ScadaDataHFD: scadaHFDResult,
			Warning:      warningResult,
			ScadaDataOEM: scadaOEMResult,
			EventDown:    eventDownResult,
			EventRaw:     eventRawResult,
			EventDownHFD: eventDownHFDResult,
			// ScadaAnomaly:      scadaAnomalyresults,
			// AlarmOverlapping:  alarmOverlappingresults,
			// AlarmScadaAnomaly: alarmScadaAnomalyresults,
		}

		mdl := new(LatestDataPeriod)

		mdl.Type = "ScadaData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaData
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "DGRData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.DGRData
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Alarm"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Alarm
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "JMR"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.JMR
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "MET"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.MET
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Duration"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Duration
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "ScadaDataHFD"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaDataHFD
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Warning"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Warning
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "ScadaDataOEM"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaDataOEM
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "EventDown"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.EventDown
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "EventRaw"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.EventRaw
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "EventDownHFD"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.EventDownHFD
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		// mdl = new(LatestDataPeriod)
		// mdl.Type = "ScadaAnomaly"
		// mdl.ProjectName = projectName
		// mdl.Data = availdatedata.ScadaAnomaly
		// mdl = mdl.New()

		// d.BaseController.Ctx.Save(mdl)

		// mdl = new(LatestDataPeriod)
		// mdl.Type = "AlarmOverlapping"
		// mdl.ProjectName = projectName
		// mdl.Data = availdatedata.AlarmOverlapping
		// mdl = mdl.New()

		// d.BaseController.Ctx.Save(mdl)

		// mdl = new(LatestDataPeriod)
		// mdl.Type = "AlarmScadaAnomaly"
		// mdl.ProjectName = projectName
		// mdl.Data = availdatedata.AlarmScadaAnomaly
		// mdl = mdl.New()

		// d.BaseController.Ctx.Save(mdl)
	}

}

func (d *GenDataPeriod) GenerateMinify(base *BaseController) {
	d.BaseController = base
	// conn, e := PrepareConnection()
	// if e != nil {
	// 	toolkit.Println("Scada Summary : " + e.Error())
	// 	os.Exit(0)
	// }
	t0 := time.Now()
	toolkit.Println("Start generating data available date : ", t0)

	var e error
	// conn := &d.BaseController.Ctx.Connection
	projects, _ := helper.GetProjectList()

	for _, proj := range projects {
		toolkit.Println("Start : ", proj.Name, " - ", t0)
		projectName := proj.Value
		scadaResults := make([]time.Time, 2)
		alarmResults := make([]time.Time, 2)
		durationResults := make([]time.Time, 2)

		scadaResults[0], scadaResults[1], e = getDataDateAvailable(d.BaseController.Ctx.Connection, new(ScadaData).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		alarmResults[0], alarmResults[1], e = getDataDateAvailable(d.BaseController.Ctx.Connection, new(Alarm).TableName(), "startdate", dbox.Eq("farm", projectName))
		durationResults[0], durationResults[1], e = getDataDateAvailable(d.BaseController.Ctx.Connection, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", false), dbox.Eq("projectname", projectName)))
		_ = e
		availdatedata := struct {
			ScadaData []time.Time
			Alarm     []time.Time
			Duration  []time.Time
		}{
			ScadaData: scadaResults,
			Alarm:     alarmResults,
			Duration:  durationResults,
		}

		mdl := new(LatestDataPeriod)

		mdl.Type = "ScadaData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaData
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Alarm"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Alarm
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Duration"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Duration
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

	}
	toolkit.Println("End generating data available date in ", time.Since(t0).String())

}

func getDataDateAvailable(conn dbox.IConnection, collectionName string, timestampColumn string, where *dbox.Filter) (min time.Time, max time.Time, err error) {
	q := conn.NewQuery().From(collectionName)

	if where != nil {
		q.Where(where)
	}

	csr, err := q.
		Aggr(dbox.AggrMin, "$"+timestampColumn, "min").
		Aggr(dbox.AggrMax, "$"+timestampColumn, "max").
		Group("enable").
		Cursor(nil)

	defer csr.Close()

	if err != nil {
		return
	}

	data := []toolkit.M{}
	err = csr.Fetch(&data, 0, false)

	if err != nil || len(data) == 0 {
		return
	}

	min = data[0].Get("min", time.Time{}).(time.Time).UTC()
	max = data[0].Get("max", time.Time{}).(time.Time).UTC()

	return
}
<commit_msg>add scada hfd with same period with scada to simplify process<commit_after>package generatorControllers

import (
	. "eaciit/wfdemo-git/library/models"
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"eaciit/wfdemo-git/web/helper"
	"time"

	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	"github.com/eaciit/toolkit"
)

type GenDataPeriod struct {
	*BaseController
}

func (d *GenDataPeriod) Generate(base *BaseController) {
	d.BaseController = base
	// conn, e := PrepareConnection()
	// if e != nil {
	// 	toolkit.Println("Scada Summary : " + e.Error())
	// 	os.Exit(0)
	// }
	// defer conn.Close()
	var e error
	conn := d.BaseController.Ctx.Connection

	// reset data
	// #faisal
	// remove this function
	// d.BaseController.Ctx.DeleteMany(NewLatestDataPeriod(), dbox.Ne("projectname", ""))

	projects, _ := helper.GetProjectList()

	for _, proj := range projects {
		projectName := proj.Value
		scadaResults := make([]time.Time, 2)
		dgrResults := make([]time.Time, 2)
		alarmResults := make([]time.Time, 2)
		jmrResults := make([]time.Time, 2)
		metResults := make([]time.Time, 2)
		durationResults := make([]time.Time, 2)
		scadaHFDResult := make([]time.Time, 2)
		warningResult := make([]time.Time, 2)
		eventDownResult := make([]time.Time, 2)
		eventRawResult := make([]time.Time, 2)
		eventDownHFDResult := make([]time.Time, 2)
		scadaOEMResult := make([]time.Time, 2)
		// scadaAnomalyresults := make([]time.Time, 2)
		// alarmOverlappingresults := make([]time.Time, 2)
		// alarmScadaAnomalyresults := make([]time.Time, 2)

		scadaResults[0], scadaResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		dgrResults[0], dgrResults[1], e = getDataDateAvailable(conn, new(DGRModel).TableName(), "dateinfo.dateid", dbox.Eq("site", projectName))
		alarmResults[0], alarmResults[1], e = getDataDateAvailable(conn, new(Alarm).TableName(), "startdate", dbox.Eq("farm", projectName))
		jmrResults[0], jmrResults[1], e = getDataDateAvailable(conn, new(JMR).TableName(), "dateinfo.dateid", nil)
		metResults[0], metResults[1], e = getDataDateAvailable(conn, new(MetTower).TableName(), "timestamp", nil)
		durationResults[0], durationResults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", false), dbox.Eq("projectname", projectName)))
		scadaHFDResult[0], scadaHFDResult[1], e = getDataDateAvailable(conn, new(ScadaDataHFD).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		warningResult[0], warningResult[1], e = getDataDateAvailable(conn, new(EventAlarm).TableName(), "timestart", dbox.Eq("projectname", projectName))
		eventDownResult[0], eventDownResult[1], e = getDataDateAvailable(conn, new(EventDown).TableName(), "timestart", dbox.Eq("projectname", projectName))
		eventRawResult[0], eventRawResult[1], e = getDataDateAvailable(conn, new(EventRaw).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		eventDownHFDResult[0], eventDownHFDResult[1], e = getDataDateAvailable(conn, new(EventDownHFD).TableName(), "timestart", dbox.Eq("projectname", projectName))
		scadaOEMResult[0], scadaOEMResult[1], e = getDataDateAvailable(conn, new(ScadaDataOEM).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		// alarmOverlappingresults[0], alarmOverlappingresults[1], e = getDataDateAvailable(conn, new(AlarmOverlapping).TableName(), "startdate", dbox.Eq("farm", projectName))
		// alarmScadaAnomalyresults[0], alarmScadaAnomalyresults[1], e = getDataDateAvailable(conn, new(AlarmScadaAnomaly).TableName(), "startdate", dbox.Eq("farm", projectName))
		// scadaAnomalyresults[0], scadaAnomalyresults[1], e = getDataDateAvailable(conn, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", true), dbox.Eq("projectname", projectName)))
		_ = e
		availdatedata := struct {
			ScadaData    []time.Time
			DGRData      []time.Time
			Alarm        []time.Time
			JMR          []time.Time
			MET          []time.Time
			Duration     []time.Time
			ScadaDataHFD []time.Time
			Warning      []time.Time
			ScadaDataOEM []time.Time
			EventDown    []time.Time
			EventRaw     []time.Time
			EventDownHFD []time.Time
			// ScadaAnomaly      []time.Time
			// AlarmOverlapping  []time.Time
			// AlarmScadaAnomaly []time.Time
		}{
			ScadaData:    scadaResults,
			DGRData:      dgrResults,
			Alarm:        alarmResults,
			JMR:          jmrResults,
			MET:          metResults,
			Duration:     durationResults,
			ScadaDataHFD: scadaHFDResult,
			Warning:      warningResult,
			ScadaDataOEM: scadaOEMResult,
			EventDown:    eventDownResult,
			EventRaw:     eventRawResult,
			EventDownHFD: eventDownHFDResult,
			// ScadaAnomaly:      scadaAnomalyresults,
			// AlarmOverlapping:  alarmOverlappingresults,
			// AlarmScadaAnomaly: alarmScadaAnomalyresults,
		}

		mdl := new(LatestDataPeriod)

		mdl.Type = "ScadaData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaData
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "DGRData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.DGRData
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Alarm"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Alarm
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "JMR"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.JMR
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "MET"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.MET
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Duration"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Duration
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "ScadaDataHFD"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaDataHFD
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Warning"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Warning
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "ScadaDataOEM"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaDataOEM
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "EventDown"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.EventDown
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "EventRaw"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.EventRaw
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "EventDownHFD"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.EventDownHFD
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		// mdl = new(LatestDataPeriod)
		// mdl.Type = "ScadaAnomaly"
		// mdl.ProjectName = projectName
		// mdl.Data = availdatedata.ScadaAnomaly
		// mdl = mdl.New()

		// d.BaseController.Ctx.Save(mdl)

		// mdl = new(LatestDataPeriod)
		// mdl.Type = "AlarmOverlapping"
		// mdl.ProjectName = projectName
		// mdl.Data = availdatedata.AlarmOverlapping
		// mdl = mdl.New()

		// d.BaseController.Ctx.Save(mdl)

		// mdl = new(LatestDataPeriod)
		// mdl.Type = "AlarmScadaAnomaly"
		// mdl.ProjectName = projectName
		// mdl.Data = availdatedata.AlarmScadaAnomaly
		// mdl = mdl.New()

		// d.BaseController.Ctx.Save(mdl)
	}

}

func (d *GenDataPeriod) GenerateMinify(base *BaseController) {
	d.BaseController = base
	// conn, e := PrepareConnection()
	// if e != nil {
	// 	toolkit.Println("Scada Summary : " + e.Error())
	// 	os.Exit(0)
	// }
	t0 := time.Now()
	toolkit.Println("Start generating data available date : ", t0)

	var e error
	// conn := &d.BaseController.Ctx.Connection
	projects, _ := helper.GetProjectList()

	for _, proj := range projects {
		toolkit.Println("Start : ", proj.Name, " - ", t0)
		projectName := proj.Value
		scadaResults := make([]time.Time, 2)
		alarmResults := make([]time.Time, 2)
		durationResults := make([]time.Time, 2)

		scadaResults[0], scadaResults[1], e = getDataDateAvailable(d.BaseController.Ctx.Connection, new(ScadaData).TableName(), "timestamp", dbox.Eq("projectname", projectName))
		alarmResults[0], alarmResults[1], e = getDataDateAvailable(d.BaseController.Ctx.Connection, new(Alarm).TableName(), "startdate", dbox.Eq("farm", projectName))
		durationResults[0], durationResults[1], e = getDataDateAvailable(d.BaseController.Ctx.Connection, new(ScadaData).TableName(), "timestamp", dbox.And(dbox.Eq("isvalidtimeduration", false), dbox.Eq("projectname", projectName)))
		_ = e
		availdatedata := struct {
			ScadaData []time.Time
			Alarm     []time.Time
			Duration  []time.Time
		}{
			ScadaData: scadaResults,
			Alarm:     alarmResults,
			Duration:  durationResults,
		}

		mdl := new(LatestDataPeriod)

		mdl.Type = "ScadaData"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaData
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)

		mdl.Type = "ScadaDataHFD"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.ScadaData
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Alarm"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Alarm
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

		mdl = new(LatestDataPeriod)
		mdl.Type = "Duration"
		mdl.ProjectName = projectName
		mdl.Data = availdatedata.Duration
		mdl = mdl.New()

		d.BaseController.Ctx.Save(mdl)

	}
	toolkit.Println("End generating data available date in ", time.Since(t0).String())

}

func getDataDateAvailable(conn dbox.IConnection, collectionName string, timestampColumn string, where *dbox.Filter) (min time.Time, max time.Time, err error) {
	q := conn.NewQuery().From(collectionName)

	if where != nil {
		q.Where(where)
	}

	csr, err := q.
		Aggr(dbox.AggrMin, "$"+timestampColumn, "min").
		Aggr(dbox.AggrMax, "$"+timestampColumn, "max").
		Group("enable").
		Cursor(nil)

	defer csr.Close()

	if err != nil {
		return
	}

	data := []toolkit.M{}
	err = csr.Fetch(&data, 0, false)

	if err != nil || len(data) == 0 {
		return
	}

	min = data[0].Get("min", time.Time{}).(time.Time).UTC()
	max = data[0].Get("max", time.Time{}).(time.Time).UTC()

	return
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/opencontainers/runc/libcontainer/utils"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/urfave/cli"
)

var execCommand = cli.Command{
	Name:  "exec",
	Usage: "execute new process inside the container",
	ArgsUsage: `<container-id> <container command>

Where "<container-id>" is the name for the instance of the container and
"<container command>" is the command to be executed in the container.

EXAMPLE:
For example, if the container is configured to run the linux ps command the
following will output a list of processes running in the container:

       # runc exec <container-id> ps`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "console",
			Usage: "specify the pty slave path for use with the container",
		},
		cli.StringFlag{
			Name:  "cwd",
			Usage: "current working directory in the container",
		},
		cli.StringSliceFlag{
			Name:  "env, e",
			Usage: "set environment variables",
		},
		cli.BoolFlag{
			Name:  "tty, t",
			Usage: "allocate a pseudo-TTY",
		},
		cli.StringFlag{
			Name:  "user, u",
			Usage: "UID (format: <uid>[:<gid>])",
		},
		cli.StringFlag{
			Name:  "process, p",
			Usage: "path to the process.json",
		},
		cli.BoolFlag{
			Name:  "detach,d",
			Usage: "detach from the container's process",
		},
		cli.StringFlag{
			Name:  "pid-file",
			Value: "",
			Usage: "specify the file to write the process id to",
		},
		cli.StringFlag{
			Name:  "process-label",
			Usage: "set the asm process label for the process commonly used with selinux",
		},
		cli.StringFlag{
			Name:  "apparmor",
			Usage: "set the apparmor profile for the process",
		},
		cli.BoolFlag{
			Name:  "no-new-privs",
			Usage: "set the no new privileges value for the process",
		},
		cli.StringSliceFlag{
			Name:  "cap, c",
			Value: &cli.StringSlice{},
			Usage: "add a capability to the bounding set for the process",
		},
		cli.BoolFlag{
			Name:  "no-subreaper",
			Usage: "disable the use of the subreaper used to reap reparented processes",
		},
	},
	Action: func(context *cli.Context) error {
		if os.Geteuid() != 0 {
			return fmt.Errorf("runc should be run as root")
		}
		status, err := execProcess(context)
		if err == nil {
			os.Exit(status)
		}
		return fmt.Errorf("exec failed: %v", err)
	},
}

func execProcess(context *cli.Context) (int, error) {
	container, err := getContainer(context)
	if err != nil {
		return -1, err
	}
	path := context.String("process")
	if path == "" && len(context.Args()) == 1 {
		return -1, fmt.Errorf("process args cannot be empty")
	}
	detach := context.Bool("detach")
	state, err := container.State()
	if err != nil {
		return -1, err
	}
	bundle := utils.SearchLabels(state.Config.Labels, "bundle")
	p, err := getProcess(context, bundle)
	if err != nil {
		return -1, err
	}
	r := &runner{
		enableSubreaper: !context.Bool("no-subreaper"),
		shouldDestroy:   false,
		container:       container,
		console:         context.String("console"),
		detach:          detach,
		pidFile:         context.String("pid-file"),
	}
	return r.run(p)
}

func getProcess(context *cli.Context, bundle string) (*specs.Process, error) {
	if path := context.String("process"); path != "" {
		f, err := os.Open(path)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		var p specs.Process
		if err := json.NewDecoder(f).Decode(&p); err != nil {
			return nil, err
		}
		return &p, validateProcessSpec(&p)
	}
	// process via cli flags
	if err := os.Chdir(bundle); err != nil {
		return nil, err
	}
	spec, err := loadSpec(specConfig)
	if err != nil {
		return nil, err
	}
	p := spec.Process
	p.Args = context.Args()[1:]
	// override the cwd, if passed
	if context.String("cwd") != "" {
		p.Cwd = context.String("cwd")
	}
	if ap := context.String("apparmor"); ap != "" {
		p.ApparmorProfile = ap
	}
	if l := context.String("process-label"); l != "" {
		p.SelinuxLabel = l
	}
	if caps := context.StringSlice("cap"); len(caps) > 0 {
		p.Capabilities = caps
	}
	// append the passed env variables
	for _, e := range context.StringSlice("env") {
		p.Env = append(p.Env, e)
	}
	// set the tty
	if context.IsSet("tty") {
		p.Terminal = context.Bool("tty")
	}
	if context.IsSet("no-new-privs") {
		p.NoNewPrivileges = context.Bool("no-new-privs")
	}
	// override the user, if passed
	if context.String("user") != "" {
		u := strings.SplitN(context.String("user"), ":", 2)
		if len(u) > 1 {
			gid, err := strconv.Atoi(u[1])
			if err != nil {
				return nil, fmt.Errorf("parsing %s as int for gid failed: %v", u[1], err)
			}
			p.User.GID = uint32(gid)
		}
		uid, err := strconv.Atoi(u[0])
		if err != nil {
			return nil, fmt.Errorf("parsing %s as int for uid failed: %v", u[0], err)
		}
		p.User.UID = uint32(uid)
	}
	return &p, nil
}
<commit_msg>Disable the subreaper on exec<commit_after>// +build linux

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/opencontainers/runc/libcontainer/utils"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/urfave/cli"
)

var execCommand = cli.Command{
	Name:  "exec",
	Usage: "execute new process inside the container",
	ArgsUsage: `<container-id> <container command>

Where "<container-id>" is the name for the instance of the container and
"<container command>" is the command to be executed in the container.

EXAMPLE:
For example, if the container is configured to run the linux ps command the
following will output a list of processes running in the container:

       # runc exec <container-id> ps`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "console",
			Usage: "specify the pty slave path for use with the container",
		},
		cli.StringFlag{
			Name:  "cwd",
			Usage: "current working directory in the container",
		},
		cli.StringSliceFlag{
			Name:  "env, e",
			Usage: "set environment variables",
		},
		cli.BoolFlag{
			Name:  "tty, t",
			Usage: "allocate a pseudo-TTY",
		},
		cli.StringFlag{
			Name:  "user, u",
			Usage: "UID (format: <uid>[:<gid>])",
		},
		cli.StringFlag{
			Name:  "process, p",
			Usage: "path to the process.json",
		},
		cli.BoolFlag{
			Name:  "detach,d",
			Usage: "detach from the container's process",
		},
		cli.StringFlag{
			Name:  "pid-file",
			Value: "",
			Usage: "specify the file to write the process id to",
		},
		cli.StringFlag{
			Name:  "process-label",
			Usage: "set the asm process label for the process commonly used with selinux",
		},
		cli.StringFlag{
			Name:  "apparmor",
			Usage: "set the apparmor profile for the process",
		},
		cli.BoolFlag{
			Name:  "no-new-privs",
			Usage: "set the no new privileges value for the process",
		},
		cli.StringSliceFlag{
			Name:  "cap, c",
			Value: &cli.StringSlice{},
			Usage: "add a capability to the bounding set for the process",
		},
		cli.BoolFlag{
			Name:   "no-subreaper",
			Usage:  "disable the use of the subreaper used to reap reparented processes",
			Hidden: true,
		},
	},
	Action: func(context *cli.Context) error {
		if os.Geteuid() != 0 {
			return fmt.Errorf("runc should be run as root")
		}
		status, err := execProcess(context)
		if err == nil {
			os.Exit(status)
		}
		return fmt.Errorf("exec failed: %v", err)
	},
}

func execProcess(context *cli.Context) (int, error) {
	container, err := getContainer(context)
	if err != nil {
		return -1, err
	}
	path := context.String("process")
	if path == "" && len(context.Args()) == 1 {
		return -1, fmt.Errorf("process args cannot be empty")
	}
	detach := context.Bool("detach")
	state, err := container.State()
	if err != nil {
		return -1, err
	}
	bundle := utils.SearchLabels(state.Config.Labels, "bundle")
	p, err := getProcess(context, bundle)
	if err != nil {
		return -1, err
	}
	r := &runner{
		enableSubreaper: false,
		shouldDestroy:   false,
		container:       container,
		console:         context.String("console"),
		detach:          detach,
		pidFile:         context.String("pid-file"),
	}
	return r.run(p)
}

func getProcess(context *cli.Context, bundle string) (*specs.Process, error) {
	if path := context.String("process"); path != "" {
		f, err := os.Open(path)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		var p specs.Process
		if err := json.NewDecoder(f).Decode(&p); err != nil {
			return nil, err
		}
		return &p, validateProcessSpec(&p)
	}
	// process via cli flags
	if err := os.Chdir(bundle); err != nil {
		return nil, err
	}
	spec, err := loadSpec(specConfig)
	if err != nil {
		return nil, err
	}
	p := spec.Process
	p.Args = context.Args()[1:]
	// override the cwd, if passed
	if context.String("cwd") != "" {
		p.Cwd = context.String("cwd")
	}
	if ap := context.String("apparmor"); ap != "" {
		p.ApparmorProfile = ap
	}
	if l := context.String("process-label"); l != "" {
		p.SelinuxLabel = l
	}
	if caps := context.StringSlice("cap"); len(caps) > 0 {
		p.Capabilities = caps
	}
	// append the passed env variables
	for _, e := range context.StringSlice("env") {
		p.Env = append(p.Env, e)
	}
	// set the tty
	if context.IsSet("tty") {
		p.Terminal = context.Bool("tty")
	}
	if context.IsSet("no-new-privs") {
		p.NoNewPrivileges = context.Bool("no-new-privs")
	}
	// override the user, if passed
	if context.String("user") != "" {
		u := strings.SplitN(context.String("user"), ":", 2)
		if len(u) > 1 {
			gid, err := strconv.Atoi(u[1])
			if err != nil {
				return nil, fmt.Errorf("parsing %s as int for gid failed: %v", u[1], err)
			}
			p.User.GID = uint32(gid)
		}
		uid, err := strconv.Atoi(u[0])
		if err != nil {
			return nil, fmt.Errorf("parsing %s as int for uid failed: %v", u[0], err)
		}
		p.User.UID = uint32(uid)
	}
	return &p, nil
}
<|endoftext|>