<commit_before>package loader

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
)

// prepare runs any special hacks for prepping the data before passing it onto
// the loader
func prepare(url, dir string) error {
	switch url {
	case "http://www.nyc.gov/html/dot/downloads/misc/siferry-gtfs.zip":
		return siFerry(dir)

	case "http://web.mta.info/developers/data/mnr/google_transit.zip":
		return mnr(dir)

	case "http://web.mta.info/developers/data/lirr/google_transit.zip":
		return lirr(dir)

	case "http://data.trilliumtransit.com/gtfs/path-nj-us/path-nj-us.zip":
		return njpath(dir)

	case "https://www.njtransit.com/mt/mt_servlet.srv?hdnPageAction=MTDevResourceDownloadTo&Category=rail":
		return njtrail(dir)

	case "http://web.mta.info/developers/data/nyct/subway/google_transit.zip":
		return mtasubway(dir)
	}

	return nil
}

type amods map[string]amod

type amod struct {
	// skip this row completely
	skip   bool
	newVal string
}

func modifyAgencies(dir string, mods amods) error {
	// appendedHeader is true if we need to append "agency_id", false
	// otherwise
	var appendedHeader bool
	var currentAgency string

	// Open up routes file as csv reader
	routeFile := path.Join(dir, "routes.txt")
	inFH, err := os.Open(routeFile)
	if err != nil {
		return err
	}
	r := csv.NewReader(inFH)
	r.LazyQuotes = true

	// Create an outgoing csv file for transformed data
	w, outFH := writecsvtmp(dir)
	defer outFH.Close()
	defer os.Remove(outFH.Name())

	// Read the existing header
	header, err := r.Read()
	if err != nil {
		return err
	}

	// Try to find agency id idx. If there is no agency_id header,
	// then add one.
	agencyIdx := maybeFind(header, "agency_id")
	if agencyIdx == -1 {
		appendedHeader = true
		header = append(header, "agency_id")
	}

	// Write header to the output file
	err = w.Write(header)
	if err != nil {
		return err
	}

	for {
		// Read until EOF or error
		rec, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if appendedHeader {
			// If there was an appended error, current value is blank
			currentAgency = ""
		} else {
			// Otherwise get the actual value
			currentAgency = rec[agencyIdx]
		}

		mod, ok := mods[currentAgency]
		// skip if requested
		if ok && mod.skip {
			continue
		}

		if ok && appendedHeader {
			// We found an empty "" => "newVal" with appended header, we must
			// append it. Basically if we're appending the header, the old
			// value must by definition be blank
			rec = append(rec, mod.newVal)

		} else if ok && !appendedHeader {
			// We found a "oldVal" => "newVal" with no need to append.
			rec[agencyIdx] = mod.newVal

		} else if !ok && appendedHeader {
			// Return an error if we didn't find a value but we needed
			// to append agency
			err = errors.New("no new value found and no existing agency")
			return err
		} else if !ok {
			// Skip if it wasn't found
			continue
		}

		// Write modified record to output
		err = w.Write(rec)
		if err != nil {
			return err
		}

	}

	// Flush and close output
	w.Flush()
	err = outFH.Close()
	if err != nil {
		return err
	}

	// Rename to official name in same dir
	err = os.Rename(outFH.Name(), routeFile)
	if err != nil {
		return err
	}

	// Success!
	return nil
}

// njpath modifies agency_id to PATH
func njpath(dir string) error {
	return modifyAgencies(dir,
		amods{
			"151": amod{false, "PATH"},
		},
	)
}

func lirr(dir string) error {
	return modifyAgencies(dir,
		amods{
			"": amod{false, "LI"},
		},
	)
}

// mnr modifies the agency id to standardize on simply "MTA MNR" rather than
// different numeric ids. Skip anything that isn't agency_id == 1.
func mnr(dir string) error {
	return modifyAgencies(dir,
		amods{
			"1": amod{false, "MTA MNR"},
		},
	)
}

// siFerry preps the Staten Island Ferry download by adding a direction id
// to files and fixing the header name of trip_headsign
func siFerry(dir string) error {
	var directionID string
	var err error

	// Get the incoming file
	tripFile := path.Join(dir, "trips.txt")
	inFH, err := os.Open(tripFile)
	if err != nil {
		return err
	}
	r := csv.NewReader(inFH)
	r.LazyQuotes = true

	// Create an outgoing csv file for transformed data
	w, outFH := writecsvtmp(dir)
	defer outFH.Close()
	defer os.Remove(outFH.Name())

	header, err := r.Read()
	if err != nil {
		return err
	}

	// Fix incorrectly named header
	for k, v := range header {
		if v == "headsign" {
			header[k] = "trip_headsign"
		}
	}

	headsignIdx := find(header, "trip_headsign")

	// Add a direction id header
	header = append(header, "direction_id")
	err = w.Write(header)
	if err != nil {
		return err
	}

	for {
		rec, err := r.Read()
		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		headsign := rec[headsignIdx]

		// SI Ferry has two destinations, add a 0/1 direction ID
		if headsign == "To Whitehall" {
			directionID = "0"
		} else {
			directionID = "1"
		}

		rec = append(rec, directionID)

		err = w.Write(rec)
		if err != nil {
			return err
		}

	}

	w.Flush()
	err = outFH.Close()
	if err != nil {
		return err
	}

	err = os.Rename(outFH.Name(), tripFile)
	if err != nil {
		return err
	}

	return nil
}

// njtrail adds route_color and route_text_color to NJT files
func njtrail(dir string) error {
	// Open up routes file as csv reader
	routeFile := path.Join(dir, "routes.txt")
	inFH, err := os.Open(routeFile)
	if err != nil {
		return err
	}
	r := csv.NewReader(inFH)
	r.LazyQuotes = true

	// Create an outgoing csv file for transformed data
	w, outFH := writecsvtmp(dir)
	defer outFH.Close()
	defer os.Remove(outFH.Name())

	// Read the existing header
	header, err := r.Read()
	if err != nil {
		return err
	}

	nameIdx := find(header, "route_long_name")
	rcIdx := find(header, "route_color")

	header = append(header, "route_text_color")
	rtIdx := len(header) - 1

	// Write header to the output file
	err = w.Write(header)
	if err != nil {
		return err
	}

	for {
		// Read until EOF or error
		rec, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		// Add one space to rec to allow for text color
		rec = append(rec, "")

		name := rec[nameIdx]
		switch name {
		case "Atlantic City Rail Line":
			rec[rcIdx] = "005DAB"
			rec[rtIdx] = "FFFFFF"

		case "Montclair-Boonton Line":
			rec[rcIdx] = "FAA634"
			rec[rtIdx] = "FFFFFF"

		case "Hudson-Bergen Light Rail", "Newark Light Rail", "Riverline Light Rail":
			rec[rcIdx] = "63F519"
			rec[rtIdx] = "000000"

		case "Main/Bergen County Line":
			rec[rcIdx] = "FFD006"
			rec[rtIdx] = "000000"

		case "Port Jervis Line":
			rec[rcIdx] = "BBCBE2"
			rec[rtIdx] = "000000"

		case "Morris & Essex Line", "Gladstone Branch":
			rec[rcIdx] = "00A850"
			rec[rtIdx] = "FFFFFF"

		case "Northeast Corridor", "Princeton Shuttle":
			rec[rcIdx] = "EE3A43"
			rec[rtIdx] = "FFFFFF"

		case "North Jersey Coast Line":
			rec[rcIdx] = "00A3E4"
			rec[rtIdx] = "000000"

		case "Pasack Valley Line":
			rec[rcIdx] = "A0218C"
			rec[rtIdx] = "FFFFFF"

		case "Raritan Valley Line":
			rec[rcIdx] = "FAA634"
			rec[rtIdx] = "000000"

		default:
			return fmt.Errorf("unrecognized NJT line: %s", name)

		}

		err = w.Write(rec)
		if err != nil {
			return err
		}
	}

	// Flush and close output
	w.Flush()
	err = outFH.Close()
	if err != nil {
		return err
	}

	// Rename to official name in same dir
	err = os.Rename(outFH.Name(), routeFile)
	if err != nil {
		return err
	}

	// Success!
	return nil
}

// mtasubway changes the color of 6X to same color as 6
func mtasubway(dir string) error {
	rw, err := newRewrite(dir, "routes.txt")
	if err != nil {
		return err
	}
	defer rw.clean()

	// Write header to the output file, unmodified
	err = rw.w.Write(rw.header)
	if err != nil {
		return err
	}

	routeIdx := find(rw.header, "route_id")
	routeColorIdx := find(rw.header, "route_color")

	for {
		// Read until EOF or error
		rec, err := rw.r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		routeID := rec[routeIdx]

		// 6X should be same color as 6
		if routeID == "6X" {
			rec[routeColorIdx] = "00933C"
		}

		err = rw.w.Write(rec)
		if err != nil {
			return err
		}
	}

	return rw.finish()
}
<commit_msg>fix nj transit pascack line<commit_after>package loader

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
)

// prepare runs any special hacks for prepping the data before passing it onto
// the loader
func prepare(url, dir string) error {
	switch url {
	case "http://www.nyc.gov/html/dot/downloads/misc/siferry-gtfs.zip":
		return siFerry(dir)

	case "http://web.mta.info/developers/data/mnr/google_transit.zip":
		return mnr(dir)

	case "http://web.mta.info/developers/data/lirr/google_transit.zip":
		return lirr(dir)

	case "http://data.trilliumtransit.com/gtfs/path-nj-us/path-nj-us.zip":
		return njpath(dir)

	case "https://www.njtransit.com/mt/mt_servlet.srv?hdnPageAction=MTDevResourceDownloadTo&Category=rail":
		return njtrail(dir)

	case "http://web.mta.info/developers/data/nyct/subway/google_transit.zip":
		return mtasubway(dir)
	}

	return nil
}

type amods map[string]amod

type amod struct {
	// skip this row completely
	skip   bool
	newVal string
}

func modifyAgencies(dir string, mods amods) error {
	// appendedHeader is true if we need to append "agency_id", false
	// otherwise
	var appendedHeader bool
	var currentAgency string

	// Open up routes file as csv reader
	routeFile := path.Join(dir, "routes.txt")
	inFH, err := os.Open(routeFile)
	if err != nil {
		return err
	}
	r := csv.NewReader(inFH)
	r.LazyQuotes = true

	// Create an outgoing csv file for transformed data
	w, outFH := writecsvtmp(dir)
	defer outFH.Close()
	defer os.Remove(outFH.Name())

	// Read the existing header
	header, err := r.Read()
	if err != nil {
		return err
	}

	// Try to find agency id idx. If there is no agency_id header,
	// then add one.
	agencyIdx := maybeFind(header, "agency_id")
	if agencyIdx == -1 {
		appendedHeader = true
		header = append(header, "agency_id")
	}

	// Write header to the output file
	err = w.Write(header)
	if err != nil {
		return err
	}

	for {
		// Read until EOF or error
		rec, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if appendedHeader {
			// If there was an appended error, current value is blank
			currentAgency = ""
		} else {
			// Otherwise get the actual value
			currentAgency = rec[agencyIdx]
		}

		mod, ok := mods[currentAgency]
		// skip if requested
		if ok && mod.skip {
			continue
		}

		if ok && appendedHeader {
			// We found an empty "" => "newVal" with appended header, we must
			// append it. Basically if we're appending the header, the old
			// value must by definition be blank
			rec = append(rec, mod.newVal)

		} else if ok && !appendedHeader {
			// We found a "oldVal" => "newVal" with no need to append.
			rec[agencyIdx] = mod.newVal

		} else if !ok && appendedHeader {
			// Return an error if we didn't find a value but we needed
			// to append agency
			err = errors.New("no new value found and no existing agency")
			return err
		} else if !ok {
			// Skip if it wasn't found
			continue
		}

		// Write modified record to output
		err = w.Write(rec)
		if err != nil {
			return err
		}

	}

	// Flush and close output
	w.Flush()
	err = outFH.Close()
	if err != nil {
		return err
	}

	// Rename to official name in same dir
	err = os.Rename(outFH.Name(), routeFile)
	if err != nil {
		return err
	}

	// Success!
	return nil
}

// njpath modifies agency_id to PATH
func njpath(dir string) error {
	return modifyAgencies(dir,
		amods{
			"151": amod{false, "PATH"},
		},
	)
}

func lirr(dir string) error {
	return modifyAgencies(dir,
		amods{
			"": amod{false, "LI"},
		},
	)
}

// mnr modifies the agency id to standardize on simply "MTA MNR" rather than
// different numeric ids. Skip anything that isn't agency_id == 1.
func mnr(dir string) error {
	return modifyAgencies(dir,
		amods{
			"1": amod{false, "MTA MNR"},
		},
	)
}

// siFerry preps the Staten Island Ferry download by adding a direction id
// to files and fixing the header name of trip_headsign
func siFerry(dir string) error {
	var directionID string
	var err error

	// Get the incoming file
	tripFile := path.Join(dir, "trips.txt")
	inFH, err := os.Open(tripFile)
	if err != nil {
		return err
	}
	r := csv.NewReader(inFH)
	r.LazyQuotes = true

	// Create an outgoing csv file for transformed data
	w, outFH := writecsvtmp(dir)
	defer outFH.Close()
	defer os.Remove(outFH.Name())

	header, err := r.Read()
	if err != nil {
		return err
	}

	// Fix incorrectly named header
	for k, v := range header {
		if v == "headsign" {
			header[k] = "trip_headsign"
		}
	}

	headsignIdx := find(header, "trip_headsign")

	// Add a direction id header
	header = append(header, "direction_id")
	err = w.Write(header)
	if err != nil {
		return err
	}

	for {
		rec, err := r.Read()
		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		headsign := rec[headsignIdx]

		// SI Ferry has two destinations, add a 0/1 direction ID
		if headsign == "To Whitehall" {
			directionID = "0"
		} else {
			directionID = "1"
		}

		rec = append(rec, directionID)

		err = w.Write(rec)
		if err != nil {
			return err
		}

	}

	w.Flush()
	err = outFH.Close()
	if err != nil {
		return err
	}

	err = os.Rename(outFH.Name(), tripFile)
	if err != nil {
		return err
	}

	return nil
}

// njtrail adds route_color and route_text_color to NJT files
func njtrail(dir string) error {
	// Open up routes file as csv reader
	routeFile := path.Join(dir, "routes.txt")
	inFH, err := os.Open(routeFile)
	if err != nil {
		return err
	}
	r := csv.NewReader(inFH)
	r.LazyQuotes = true

	// Create an outgoing csv file for transformed data
	w, outFH := writecsvtmp(dir)
	defer outFH.Close()
	defer os.Remove(outFH.Name())

	// Read the existing header
	header, err := r.Read()
	if err != nil {
		return err
	}

	nameIdx := find(header, "route_long_name")
	rcIdx := find(header, "route_color")

	header = append(header, "route_text_color")
	rtIdx := len(header) - 1

	// Write header to the output file
	err = w.Write(header)
	if err != nil {
		return err
	}

	for {
		// Read until EOF or error
		rec, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		// Add one space to rec to allow for text color
		rec = append(rec, "")

		name := rec[nameIdx]
		switch name {
		case "Atlantic City Rail Line":
			rec[rcIdx] = "005DAB"
			rec[rtIdx] = "FFFFFF"

		case "Montclair-Boonton Line":
			rec[rcIdx] = "FAA634"
			rec[rtIdx] = "FFFFFF"

		case "Hudson-Bergen Light Rail", "Newark Light Rail", "Riverline Light Rail":
			rec[rcIdx] = "63F519"
			rec[rtIdx] = "000000"

		case "Main/Bergen County Line":
			rec[rcIdx] = "FFD006"
			rec[rtIdx] = "000000"

		case "Port Jervis Line":
			rec[rcIdx] = "BBCBE2"
			rec[rtIdx] = "000000"

		case "Morris & Essex Line", "Gladstone Branch":
			rec[rcIdx] = "00A850"
			rec[rtIdx] = "FFFFFF"

		case "Northeast Corridor", "Princeton Shuttle":
			rec[rcIdx] = "EE3A43"
			rec[rtIdx] = "FFFFFF"

		case "North Jersey Coast Line":
			rec[rcIdx] = "00A3E4"
			rec[rtIdx] = "000000"

		case "Pascack Valley Line":
			rec[rcIdx] = "A0218C"
			rec[rtIdx] = "FFFFFF"

		case "Raritan Valley Line":
			rec[rcIdx] = "FAA634"
			rec[rtIdx] = "000000"

		default:
			return fmt.Errorf("unrecognized NJT line: %s", name)

		}

		err = w.Write(rec)
		if err != nil {
			return err
		}
	}

	// Flush and close output
	w.Flush()
	err = outFH.Close()
	if err != nil {
		return err
	}

	// Rename to official name in same dir
	err = os.Rename(outFH.Name(), routeFile)
	if err != nil {
		return err
	}

	// Success!
	return nil
}

// mtasubway changes the color of 6X to same color as 6
func mtasubway(dir string) error {
	rw, err := newRewrite(dir, "routes.txt")
	if err != nil {
		return err
	}
	defer rw.clean()

	// Write header to the output file, unmodified
	err = rw.w.Write(rw.header)
	if err != nil {
		return err
	}

	routeIdx := find(rw.header, "route_id")
	routeColorIdx := find(rw.header, "route_color")

	for {
		// Read until EOF or error
		rec, err := rw.r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		routeID := rec[routeIdx]

		// 6X should be same color as 6
		if routeID == "6X" {
			rec[routeColorIdx] = "00933C"
		}

		err = rw.w.Write(rec)
		if err != nil {
			return err
		}
	}

	return rw.finish()
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package images

import (
	"time"

	"github.com/pkg/errors"
)

var (
	ErrModelMissingInputMetadata     = errors.New("Missing input metadata")
	ErrModelImageInActiveDeployment  = errors.New("Image is used in active deployment and cannot be removed")
	ErrModelImageUsedInAnyDeployment = errors.New("Image have been already used in deployment")
)

type SoftwareImagesStorager interface {
	Exists(id string) (bool, error)
	Update(image *SoftwareImage) (bool, error)
	Insert(image *SoftwareImage) error
	FindByID(id string) (*SoftwareImage, error)
	Delete(id string) error
	FindAll() ([]*SoftwareImage, error)
}

type FileStorager interface {
	Delete(objectId string) error
	Exists(objectId string) (bool, error)
	LastModified(objectId string) (time.Time, error)
	PutRequest(objectId string, duration time.Duration) (*Link, error)
	GetRequest(objectId string, duration time.Duration) (*Link, error)
}

type ImageInUseChecker interface {
	ImageUsedInActiveDeployment(imageId string) (bool, error)
	ImageUsedInDeployment(imageId string) (bool, error)
}

type ImagesModel struct {
	fileStorage   FileStorager
	deployments   ImageInUseChecker
	imagesStorage SoftwareImagesStorager
}

func NewImagesModel(
	fileStorage FileStorager,
	checker ImageInUseChecker,
	imagesStorage SoftwareImagesStorager,
) *ImagesModel {
	return &ImagesModel{
		fileStorage:   fileStorage,
		deployments:   checker,
		imagesStorage: imagesStorage,
	}
}

func (i *ImagesModel) CreateImage(constructor *SoftwareImageConstructor) (string, error) {

	if constructor == nil {
		return "", ErrModelMissingInputMetadata
	}

	if err := constructor.Validate(); err != nil {
		return "", errors.Wrap(err, "Validating image metadata")
	}

	image := NewSoftwareImageFromConstructor(constructor)

	if err := i.imagesStorage.Insert(image); err != nil {
		return "", errors.Wrap(err, "Storing image metadata")
	}

	return *image.Id, nil
}

// GetImage allows to fetch image obeject with specified id
// On each lookup it syncs last file upload time with metadata in case file was uploaded or reuploaded
// Nil if not found
func (i *ImagesModel) GetImage(id string) (*SoftwareImage, error) {

	image, err := i.imagesStorage.FindByID(id)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if image == nil {
		return nil, nil
	}

	if err := i.syncLastModifiedTimeWithFileUpload(image); err != nil {
		return nil, errors.Wrap(err, "Synchronizing image upload time")
	}

	return image, nil
}

// DeleteImage removes metadata and image file
// Noop for not exisitng images
// Allowed to remove image only if image is not scheduled or in progress for an updates - then image file is needed
// In case of already finished updates only image file is not needed, metadata is attached directly to device deployment
// therefore we still have some information about image that have been used (but not the file)
func (i *ImagesModel) DeleteImage(imageID string) error {

	inUse, err := i.deployments.ImageUsedInActiveDeployment(imageID)
	if err != nil {
		return errors.Wrap(err, "Checking if image is used in active deployment")
	}

	// Image is in use, not allowed to delete
	if inUse {
		return ErrModelImageInActiveDeployment
	}

	// Delete image file (call to external service)
	// Noop for not existing file
	if err := i.fileStorage.Delete(imageID); err != nil {
		return errors.Wrap(err, "Deleting image file")
	}

	// Delete metadata
	if err := i.imagesStorage.Delete(imageID); err != nil {
		return errors.Wrap(err, "Deleting image metadata")
	}

	return nil
}

// ListImages according to specified filers.
// On each lookup it syncs last file upload time with metadata in case file was uploaded or reuploaded
func (i *ImagesModel) ListImages(filters map[string]string) ([]*SoftwareImage, error) {

	images, err := i.imagesStorage.FindAll()
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image metadata")
	}

	for _, image := range images {
		if err := i.syncLastModifiedTimeWithFileUpload(image); err != nil {
			return nil, errors.Wrap(err, "Synchronizing image upload time")
		}
	}

	return images, nil
}

// Sync file upload time with last modified time of image metadata.
// Need to check when image was uploaded and if it was overwritten
// Ugly but required by frontend design, in future can be split.
// Expensive! Will go away with switching to one-step file upload.
func (i *ImagesModel) syncLastModifiedTimeWithFileUpload(image *SoftwareImage) error {

	uploaded, err := i.fileStorage.LastModified(*image.Id)

	if err != nil && errors.Cause(err).Error() == ErrFileStorageFileNotFound.Error() {
		return nil
	}

	if err != nil {
		return errors.Wrap(err, "Cheking last modified time for image file")
	}

	if image.Modified.Before(uploaded) {
		image.Modified = &uploaded
		if _, err := i.imagesStorage.Update(image); err != nil {
			return errors.Wrap(err, "Updating image metadata")
		}
	}

	return nil
}

// EditObject allows editing only if image have not been used yet in any deployment.
func (i *ImagesModel) EditImage(imageID string, constructor *SoftwareImageConstructor) (bool, error) {

	if err := constructor.Validate(); err != nil {
		return false, errors.Wrap(err, "Validating image metadata")
	}

	found, err := i.deployments.ImageUsedInDeployment(imageID)
	if err != nil {
		return false, errors.Wrap(err, "Searching for usage of the image among deployments")
	}

	if found {
		return false, ErrModelImageUsedInAnyDeployment
	}

	foundImage, err := i.imagesStorage.FindByID(imageID)
	if err != nil {
		return false, errors.Wrap(err, "Searching for image with specified ID")
	}

	if foundImage == nil {
		return false, nil
	}

	foundImage.SoftwareImageConstructor = constructor
	foundImage.SetModified(time.Now())

	_, err = i.imagesStorage.Update(foundImage)
	if err != nil {
		return false, errors.Wrap(err, "Updating image matadata")
	}

	return true, nil
}

// UploadLink generated presigned PUT link to upload image file.
// Image meta has to be created first.
func (i *ImagesModel) UploadLink(imageID string, expire time.Duration) (*Link, error) {

	found, err := i.imagesStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if !found {
		return nil, nil
	}

	link, err := i.fileStorage.PutRequest(imageID, expire)
	if err != nil {
		return nil, errors.Wrap(err, "Generating upload link")
	}

	return link, nil
}

// DownloadLink presigned GET link to download image file.
// Returns error if image have not been uploaded.
func (i *ImagesModel) DownloadLink(imageID string, expire time.Duration) (*Link, error) {

	found, err := i.imagesStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if !found {
		return nil, nil
	}

	found, err = i.fileStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image file")
	}

	if !found {
		return nil, nil
	}

	link, err := i.fileStorage.GetRequest(imageID, expire)
	if err != nil {
		return nil, errors.Wrap(err, "Generating download link")
	}

	return link, nil
}
<commit_msg>Tidy-up error handling in syncLastModifiedTimeWithFileUpload<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package images

import (
	"time"

	"github.com/pkg/errors"
)

var (
	ErrModelMissingInputMetadata     = errors.New("Missing input metadata")
	ErrModelImageInActiveDeployment  = errors.New("Image is used in active deployment and cannot be removed")
	ErrModelImageUsedInAnyDeployment = errors.New("Image have been already used in deployment")
)

type SoftwareImagesStorager interface {
	Exists(id string) (bool, error)
	Update(image *SoftwareImage) (bool, error)
	Insert(image *SoftwareImage) error
	FindByID(id string) (*SoftwareImage, error)
	Delete(id string) error
	FindAll() ([]*SoftwareImage, error)
}

type FileStorager interface {
	Delete(objectId string) error
	Exists(objectId string) (bool, error)
	LastModified(objectId string) (time.Time, error)
	PutRequest(objectId string, duration time.Duration) (*Link, error)
	GetRequest(objectId string, duration time.Duration) (*Link, error)
}

type ImageInUseChecker interface {
	ImageUsedInActiveDeployment(imageId string) (bool, error)
	ImageUsedInDeployment(imageId string) (bool, error)
}

type ImagesModel struct {
	fileStorage   FileStorager
	deployments   ImageInUseChecker
	imagesStorage SoftwareImagesStorager
}

func NewImagesModel(
	fileStorage FileStorager,
	checker ImageInUseChecker,
	imagesStorage SoftwareImagesStorager,
) *ImagesModel {
	return &ImagesModel{
		fileStorage:   fileStorage,
		deployments:   checker,
		imagesStorage: imagesStorage,
	}
}

func (i *ImagesModel) CreateImage(constructor *SoftwareImageConstructor) (string, error) {

	if constructor == nil {
		return "", ErrModelMissingInputMetadata
	}

	if err := constructor.Validate(); err != nil {
		return "", errors.Wrap(err, "Validating image metadata")
	}

	image := NewSoftwareImageFromConstructor(constructor)

	if err := i.imagesStorage.Insert(image); err != nil {
		return "", errors.Wrap(err, "Storing image metadata")
	}

	return *image.Id, nil
}

// GetImage allows to fetch image obeject with specified id
// On each lookup it syncs last file upload time with metadata in case file was uploaded or reuploaded
// Nil if not found
func (i *ImagesModel) GetImage(id string) (*SoftwareImage, error) {

	image, err := i.imagesStorage.FindByID(id)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if image == nil {
		return nil, nil
	}

	if err := i.syncLastModifiedTimeWithFileUpload(image); err != nil {
		return nil, errors.Wrap(err, "Synchronizing image upload time")
	}

	return image, nil
}

// DeleteImage removes metadata and image file
// Noop for not exisitng images
// Allowed to remove image only if image is not scheduled or in progress for an updates - then image file is needed
// In case of already finished updates only image file is not needed, metadata is attached directly to device deployment
// therefore we still have some information about image that have been used (but not the file)
func (i *ImagesModel) DeleteImage(imageID string) error {

	inUse, err := i.deployments.ImageUsedInActiveDeployment(imageID)
	if err != nil {
		return errors.Wrap(err, "Checking if image is used in active deployment")
	}

	// Image is in use, not allowed to delete
	if inUse {
		return ErrModelImageInActiveDeployment
	}

	// Delete image file (call to external service)
	// Noop for not existing file
	if err := i.fileStorage.Delete(imageID); err != nil {
		return errors.Wrap(err, "Deleting image file")
	}

	// Delete metadata
	if err := i.imagesStorage.Delete(imageID); err != nil {
		return errors.Wrap(err, "Deleting image metadata")
	}

	return nil
}

// ListImages according to specified filers.
// On each lookup it syncs last file upload time with metadata in case file was uploaded or reuploaded
func (i *ImagesModel) ListImages(filters map[string]string) ([]*SoftwareImage, error) {

	images, err := i.imagesStorage.FindAll()
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image metadata")
	}

	for _, image := range images {
		if err := i.syncLastModifiedTimeWithFileUpload(image); err != nil {
			return nil, errors.Wrap(err, "Synchronizing image upload time")
		}
	}

	return images, nil
}

// Sync file upload time with last modified time of image metadata.
// Need to check when image was uploaded and if it was overwritten
// Ugly but required by frontend design, in future can be split.
// Expensive! Will go away with switching to one-step file upload.
func (i *ImagesModel) syncLastModifiedTimeWithFileUpload(image *SoftwareImage) error {

	uploaded, err := i.fileStorage.LastModified(*image.Id)
	if err != nil {
		if errors.Cause(err).Error() == ErrFileStorageFileNotFound.Error() {
			return nil
		}

		return errors.Wrap(err, "Cheking last modified time for image file")
	}

	if image.Modified.Before(uploaded) {
		image.Modified = &uploaded
		if _, err := i.imagesStorage.Update(image); err != nil {
			return errors.Wrap(err, "Updating image metadata")
		}
	}

	return nil
}

// EditObject allows editing only if image have not been used yet in any deployment.
func (i *ImagesModel) EditImage(imageID string, constructor *SoftwareImageConstructor) (bool, error) {

	if err := constructor.Validate(); err != nil {
		return false, errors.Wrap(err, "Validating image metadata")
	}

	found, err := i.deployments.ImageUsedInDeployment(imageID)
	if err != nil {
		return false, errors.Wrap(err, "Searching for usage of the image among deployments")
	}

	if found {
		return false, ErrModelImageUsedInAnyDeployment
	}

	foundImage, err := i.imagesStorage.FindByID(imageID)
	if err != nil {
		return false, errors.Wrap(err, "Searching for image with specified ID")
	}

	if foundImage == nil {
		return false, nil
	}

	foundImage.SoftwareImageConstructor = constructor
	foundImage.SetModified(time.Now())

	_, err = i.imagesStorage.Update(foundImage)
	if err != nil {
		return false, errors.Wrap(err, "Updating image matadata")
	}

	return true, nil
}

// UploadLink generated presigned PUT link to upload image file.
// Image meta has to be created first.
func (i *ImagesModel) UploadLink(imageID string, expire time.Duration) (*Link, error) {

	found, err := i.imagesStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if !found {
		return nil, nil
	}

	link, err := i.fileStorage.PutRequest(imageID, expire)
	if err != nil {
		return nil, errors.Wrap(err, "Generating upload link")
	}

	return link, nil
}

// DownloadLink presigned GET link to download image file.
// Returns error if image have not been uploaded.
func (i *ImagesModel) DownloadLink(imageID string, expire time.Duration) (*Link, error) {

	found, err := i.imagesStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if !found {
		return nil, nil
	}

	found, err = i.fileStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image file")
	}

	if !found {
		return nil, nil
	}

	link, err := i.fileStorage.GetRequest(imageID, expire)
	if err != nil {
		return nil, errors.Wrap(err, "Generating download link")
	}

	return link, nil
}
<|endoftext|>


<commit_before>package posix

import (
	"os"
	"strings"
	"syscall"

	co "github.com/lunixbochs/usercorn/go/kernel/common"
)

func (k *PosixKernel) Read(fd co.Fd, buf co.Obuf, size co.Len) uint64 {
	tmp := make([]byte, size)
	n, err := syscall.Read(int(fd), tmp)
	if err != nil {
		return Errno(err)
	}
	if err := buf.Pack(tmp[:n]); err != nil {
		return UINT64_MAX // FIXME
	}
	return uint64(n)
}

func (k *PosixKernel) Write(fd co.Fd, buf co.Buf, size co.Len) uint64 {
	tmp := make([]byte, size)
	if err := buf.Unpack(tmp); err != nil {
		return UINT64_MAX // FIXME
	}
	n, err := syscall.Write(int(fd), tmp)
	if err != nil {
		return Errno(err)
	}
	return uint64(n)
}

func (k *PosixKernel) Open(path string, mode int, flags uint32) uint64 {
	if strings.Contains(path, "/lib/") {
		path = k.U.PrefixPath(path, false)
	}
	fd, err := syscall.Open(path, mode, flags)
	if err != nil {
		return Errno(err)
	}
	return uint64(fd)
}

func (k *PosixKernel) Close(fd co.Fd) uint64 {
	// FIXME: temporary hack to preserve output on program exit
	if fd == 2 {
		return 0
	}
	return Errno(syscall.Close(int(fd)))
}

func (k *PosixKernel) Lseek(fd co.Fd, offset co.Off, whence int) uint64 {
	off, err := syscall.Seek(int(fd), int64(offset), whence)
	if err != nil {
		return Errno(err)
	}
	return uint64(off)
}

func (k *PosixKernel) Fstat(fd co.Fd, buf co.Buf) uint64 {
	var stat syscall.Stat_t
	if err := syscall.Fstat(int(fd), &stat); err != nil {
		return Errno(err)
	}
	targetStat := NewTargetStat(&stat, k.U.OS(), k.U.Bits())
	if err := buf.Pack(targetStat); err != nil {
		panic(err)
	}
	return 0
}

func (k *PosixKernel) Lstat(path string, buf co.Buf) uint64 {
	var stat syscall.Stat_t
	if err := syscall.Lstat(path, &stat); err != nil {
		return Errno(err)
	}
	targetStat := NewTargetStat(&stat, k.U.OS(), k.U.Bits())
	if err := buf.Pack(targetStat); err != nil {
		panic(err)
	}
	return 0
}

func (k *PosixKernel) Stat(path string, buf co.Buf) uint64 {
	// TODO: centralize path hook
	if strings.Contains(path, "/lib/") {
		path = k.U.PrefixPath(path, false)
	}
	var stat syscall.Stat_t
	if err := syscall.Stat(path, &stat); err != nil {
		return Errno(err)
	}
	targetStat := NewTargetStat(&stat, k.U.OS(), k.U.Bits())
	if err := buf.Pack(targetStat); err != nil {
		panic(err)
	}
	return 0
}

func (k *PosixKernel) Getcwd(buf co.Buf, size co.Len) uint64 {
	wd, _ := os.Getwd()
	size -= 1
	if co.Len(len(wd)) > size {
		wd = wd[:size]
	}
	if err := buf.Pack(wd + "\x00"); err != nil {
		return UINT64_MAX // FIXME
	}
	return 0
}

func (k *PosixKernel) Access(path string, amode uint32) uint64 {
	// TODO: portability
	return Errno(syscall.Access(path, amode))
}

func (k *PosixKernel) Readv(fd co.Fd, iov co.Buf, count uint64) uint64 {
	var read uint64
	for vec := range iovecIter(iov, count, k.U.Bits()) {
		tmp := make([]byte, vec.Len)
		n, err := syscall.Read(int(fd), tmp)
		if err != nil {
			return Errno(err)
		}
		read += uint64(n)
		k.U.MemWrite(vec.Base, tmp[:n])
	}
	return read
}

func (k *PosixKernel) Writev(fd co.Fd, iov co.Buf, count uint64) uint64 {
	var written uint64
	for vec := range iovecIter(iov, count, k.U.Bits()) {
		data, _ := k.U.MemRead(vec.Base, vec.Len)
		n, err := syscall.Write(int(fd), data)
		if err != nil {
			return Errno(err)
		}
		written += uint64(n)
	}
	return written
}

func (k *PosixKernel) Pread64(fd co.Fd, buf co.Obuf, size co.Len, offset int64) uint64 {
	p := make([]byte, size)
	n, err := syscall.Pread(int(fd), p, offset)
	if err != nil {
		return Errno(err)
	}
	if err := buf.Pack(p); err != nil {
		return UINT64_MAX // FIXME
	}
	return uint64(n)
}

func (k *PosixKernel) Pwrite64(fd co.Fd, buf co.Buf, size co.Len, offset int64) uint64 {
	p := make([]byte, size)
	if err := buf.Unpack(p); err != nil {
		return UINT64_MAX // FIXME
	}
	n, err := syscall.Pwrite(int(fd), p, offset)
	if err != nil {
		return Errno(err)
	}
	return uint64(n)
}

func (k *PosixKernel) Chmod(path string, mode uint32) uint64 {
	return Errno(syscall.Chmod(path, mode))
}

func (k *PosixKernel) Fchmod(fd int, mode uint32) uint64 {
	return Errno(syscall.Fchmod(fd, mode))
}

func (k *PosixKernel) Chown(path string, uid, gid int) uint64 {
	return Errno(syscall.Chown(path, uid, gid))
}

func (k *PosixKernel) Fchown(fd, uid, gid int) uint64 {
	return Errno(syscall.Fchown(fd, uid, gid))
}

func (k *PosixKernel) Lchown(path string, uid, gid int) uint64 {
	return Errno(syscall.Lchown(path, uid, gid))
}

func (k *PosixKernel) Dup(oldFd co.Fd) uint64 {
	if newFd, err := syscall.Dup(int(oldFd)); err != nil {
		return Errno(err)
	} else {
		return uint64(newFd)
	}
}

func (k *PosixKernel) Dup2(oldFd co.Fd, newFd co.Fd) uint64 {
	if err := syscall.Dup2(int(oldFd), int(newFd)); err != nil {
		return Errno(err)
	}
	return uint64(newFd)
}

func (k *PosixKernel) Readlink(path string, buf co.Obuf, size co.Len) uint64 {
	// TODO: full proc emulation layer
	// maybe have a syscall pre-hook for this after ghostrace makes it generic
	// or specifically have path hooks and use that to implement prefix as well
	var name string
	var err error
	if path == "/proc/self/exe" && k.U.OS() == "linux" {
		name = k.U.Exe()
	} else {
		name, err = os.Readlink(path)
		if err != nil {
			return UINT64_MAX // FIXME
		}
	}
	if len(name) > int(size)-1 {
		name = name[:size-1]
	}
	if err := buf.Pack([]byte(name + "\x00")); err != nil {
		return UINT64_MAX // FIXME
	}
	return uint64(len(name))
}

func (k *PosixKernel) Symlink(src, dst string) uint64 {
	return Errno(syscall.Symlink(src, dst))
}

func (k *PosixKernel) Link(src, dst string) uint64 {
	return Errno(syscall.Link(src, dst))
}

func (k *PosixKernel) Openat(dirfd co.Fd, path string, flags int, mode uint32) uint64 {
	// TODO: flags might be different per arch
	return openat_native(int(dirfd), path, flags, mode)
}

func (k *PosixKernel) Chdir(path string) uint64 {
	if err := os.Chdir(path); err != nil {
		return UINT64_MAX // FIXME
	}
	return 0
}

func (k *PosixKernel) Chroot(path string) uint64 {
	return Errno(syscall.Chroot(path))
}

func (k *PosixKernel) Pipe(files co.Buf) uint64 {
	var fds [2]int
	err := syscall.Pipe(fds[:])
	if err == nil {
		st := files.Struc()
		err := st.Pack(int32(fds[0]))
		if err == nil {
			err = st.Pack(int32(fds[1]))
		}
		if err != nil {
			return UINT64_MAX // FIXME
		}
	}
	return Errno(err)
}

func (k *PosixKernel) Pipe2(files co.Buf, flags int) uint64 {
	// TODO: handle flags
	return k.Pipe(files)
}
<commit_msg>fix issue with open syscall not converting args<commit_after>package posix

import (
	"os"
	"strings"
	"syscall"

	co "github.com/lunixbochs/usercorn/go/kernel/common"
)

func (k *PosixKernel) Read(fd co.Fd, buf co.Obuf, size co.Len) uint64 {
	tmp := make([]byte, size)
	n, err := syscall.Read(int(fd), tmp)
	if err != nil {
		return Errno(err)
	}
	if err := buf.Pack(tmp[:n]); err != nil {
		return UINT64_MAX // FIXME
	}
	return uint64(n)
}

func (k *PosixKernel) Write(fd co.Fd, buf co.Buf, size co.Len) uint64 {
	tmp := make([]byte, size)
	if err := buf.Unpack(tmp); err != nil {
		return UINT64_MAX // FIXME
	}
	n, err := syscall.Write(int(fd), tmp)
	if err != nil {
		return Errno(err)
	}
	return uint64(n)
}

func (k *PosixKernel) Open(path string, mode int, flags uint64) uint64 {
	if strings.Contains(path, "/lib/") {
		path = k.U.PrefixPath(path, false)
	}
	fd, err := syscall.Open(path, mode, uint32(flags))
	if err != nil {
		return Errno(err)
	}
	return uint64(fd)
}

func (k *PosixKernel) Close(fd co.Fd) uint64 {
	// FIXME: temporary hack to preserve output on program exit
	if fd == 2 {
		return 0
	}
	return Errno(syscall.Close(int(fd)))
}

func (k *PosixKernel) Lseek(fd co.Fd, offset co.Off, whence int) uint64 {
	off, err := syscall.Seek(int(fd), int64(offset), whence)
	if err != nil {
		return Errno(err)
	}
	return uint64(off)
}

func (k *PosixKernel) Fstat(fd co.Fd, buf co.Buf) uint64 {
	var stat syscall.Stat_t
	if err := syscall.Fstat(int(fd), &stat); err != nil {
		return Errno(err)
	}
	targetStat := NewTargetStat(&stat, k.U.OS(), k.U.Bits())
	if err := buf.Pack(targetStat); err != nil {
		panic(err)
	}
	return 0
}

func (k *PosixKernel) Lstat(path string, buf co.Buf) uint64 {
	var stat syscall.Stat_t
	if err := syscall.Lstat(path, &stat); err != nil {
		return Errno(err)
	}
	targetStat := NewTargetStat(&stat, k.U.OS(), k.U.Bits())
	if err := buf.Pack(targetStat); err != nil {
		panic(err)
	}
	return 0
}

func (k *PosixKernel) Stat(path string, buf co.Buf) uint64 {
	// TODO: centralize path hook
	if strings.Contains(path, "/lib/") {
		path = k.U.PrefixPath(path, false)
	}
	var stat syscall.Stat_t
	if err := syscall.Stat(path, &stat); err != nil {
		return Errno(err)
	}
	targetStat := NewTargetStat(&stat, k.U.OS(), k.U.Bits())
	if err := buf.Pack(targetStat); err != nil {
		panic(err)
	}
	return 0
}

func (k *PosixKernel) Getcwd(buf co.Buf, size co.Len) uint64 {
	wd, _ := os.Getwd()
	size -= 1
	if co.Len(len(wd)) > size {
		wd = wd[:size]
	}
	if err := buf.Pack(wd + "\x00"); err != nil {
		return UINT64_MAX // FIXME
	}
	return 0
}

func (k *PosixKernel) Access(path string, amode uint32) uint64 {
	// TODO: portability
	return Errno(syscall.Access(path, amode))
}

func (k *PosixKernel) Readv(fd co.Fd, iov co.Buf, count uint64) uint64 {
	var read uint64
	for vec := range iovecIter(iov, count, k.U.Bits()) {
		tmp := make([]byte, vec.Len)
		n, err := syscall.Read(int(fd), tmp)
		if err != nil {
			return Errno(err)
		}
		read += uint64(n)
		k.U.MemWrite(vec.Base, tmp[:n])
	}
	return read
}

func (k *PosixKernel) Writev(fd co.Fd, iov co.Buf, count uint64) uint64 {
	var written uint64
	for vec := range iovecIter(iov, count, k.U.Bits()) {
		data, _ := k.U.MemRead(vec.Base, vec.Len)
		n, err := syscall.Write(int(fd), data)
		if err != nil {
			return Errno(err)
		}
		written += uint64(n)
	}
	return written
}

func (k *PosixKernel) Pread64(fd co.Fd, buf co.Obuf, size co.Len, offset int64) uint64 {
	p := make([]byte, size)
	n, err := syscall.Pread(int(fd), p, offset)
	if err != nil {
		return Errno(err)
	}
	if err := buf.Pack(p); err != nil {
		return UINT64_MAX // FIXME
	}
	return uint64(n)
}

func (k *PosixKernel) Pwrite64(fd co.Fd, buf co.Buf, size co.Len, offset int64) uint64 {
	p := make([]byte, size)
	if err := buf.Unpack(p); err != nil {
		return UINT64_MAX // FIXME
	}
	n, err := syscall.Pwrite(int(fd), p, offset)
	if err != nil {
		return Errno(err)
	}
	return uint64(n)
}

func (k *PosixKernel) Chmod(path string, mode uint32) uint64 {
	return Errno(syscall.Chmod(path, mode))
}

func (k *PosixKernel) Fchmod(fd int, mode uint32) uint64 {
	return Errno(syscall.Fchmod(fd, mode))
}

func (k *PosixKernel) Chown(path string, uid, gid int) uint64 {
	return Errno(syscall.Chown(path, uid, gid))
}

func (k *PosixKernel) Fchown(fd, uid, gid int) uint64 {
	return Errno(syscall.Fchown(fd, uid, gid))
}

func (k *PosixKernel) Lchown(path string, uid, gid int) uint64 {
	return Errno(syscall.Lchown(path, uid, gid))
}

func (k *PosixKernel) Dup(oldFd co.Fd) uint64 {
	if newFd, err := syscall.Dup(int(oldFd)); err != nil {
		return Errno(err)
	} else {
		return uint64(newFd)
	}
}

func (k *PosixKernel) Dup2(oldFd co.Fd, newFd co.Fd) uint64 {
	if err := syscall.Dup2(int(oldFd), int(newFd)); err != nil {
		return Errno(err)
	}
	return uint64(newFd)
}

func (k *PosixKernel) Readlink(path string, buf co.Obuf, size co.Len) uint64 {
	// TODO: full proc emulation layer
	// maybe have a syscall pre-hook for this after ghostrace makes it generic
	// or specifically have path hooks and use that to implement prefix as well
	var name string
	var err error
	if path == "/proc/self/exe" && k.U.OS() == "linux" {
		name = k.U.Exe()
	} else {
		name, err = os.Readlink(path)
		if err != nil {
			return UINT64_MAX // FIXME
		}
	}
	if len(name) > int(size)-1 {
		name = name[:size-1]
	}
	if err := buf.Pack([]byte(name + "\x00")); err != nil {
		return UINT64_MAX // FIXME
	}
	return uint64(len(name))
}

func (k *PosixKernel) Symlink(src, dst string) uint64 {
	return Errno(syscall.Symlink(src, dst))
}

func (k *PosixKernel) Link(src, dst string) uint64 {
	return Errno(syscall.Link(src, dst))
}

func (k *PosixKernel) Openat(dirfd co.Fd, path string, flags int, mode uint32) uint64 {
	// TODO: flags might be different per arch
	return openat_native(int(dirfd), path, flags, mode)
}

func (k *PosixKernel) Chdir(path string) uint64 {
	if err := os.Chdir(path); err != nil {
		return UINT64_MAX // FIXME
	}
	return 0
}

func (k *PosixKernel) Chroot(path string) uint64 {
	return Errno(syscall.Chroot(path))
}

func (k *PosixKernel) Pipe(files co.Buf) uint64 {
	var fds [2]int
	err := syscall.Pipe(fds[:])
	if err == nil {
		st := files.Struc()
		err := st.Pack(int32(fds[0]))
		if err == nil {
			err = st.Pack(int32(fds[1]))
		}
		if err != nil {
			return UINT64_MAX // FIXME
		}
	}
	return Errno(err)
}

func (k *PosixKernel) Pipe2(files co.Buf, flags int) uint64 {
	// TODO: handle flags
	return k.Pipe(files)
}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"errors"
	"net/http"
	"runtime"
	"testing"
	"time"

	"github.com/ant0ine/go-json-rest/rest"
	"github.com/ant0ine/go-json-rest/rest/test"
	"github.com/mendersoftware/artifacts/controllers"
	"github.com/mendersoftware/artifacts/models/images"
	"github.com/mendersoftware/artifacts/models/users"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

type SetUserMiddleware struct{}

func (mw *SetUserMiddleware) MiddlewareFunc(handler rest.HandlerFunc) rest.HandlerFunc {
	return func(writer rest.ResponseWriter, request *rest.Request) {
		request.Env["REMOTE_USER"] = "admin"
		handler(writer, request)
	}
}

func TestImageMetaLookup(t *testing.T) {

	mock := &ImageControllerMock{
		mockLookup: func(user users.UserI) ([]*images.ImageMeta, error) {
			return make([]*images.ImageMeta, 0), nil
		},
	}

	router, err := rest.MakeRouter(rest.Get("/r", NewImageMeta(mock).Lookup))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("GET", "http://1.2.3.4/r", nil))

	recorded.CodeIs(http.StatusOK)
	recorded.ContentTypeIsJson()
	recorded.BodyIs("[]")
}

func TestImageMetaLookupError(t *testing.T) {

	apiErr := errors.New("My dummy error")

	mock := &ImageControllerMock{
		mockLookup: func(user users.UserI) ([]*images.ImageMeta, error) {
			return nil, apiErr
		},
	}

	router, err := rest.MakeRouter(rest.Get("/r", NewImageMeta(mock).Lookup))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("GET", "http://1.2.3.4/r", nil))

	recorded.CodeIs(http.StatusInternalServerError)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(apiErr))
}

func TestImageMetaGetNotFound(t *testing.T) {

	mock := &ImageControllerMock{
		mockGet: func(user users.UserI, id string) (*images.ImageMeta, error) {
			return nil, controllers.ErrNotFound
		},
	}

	router, err := rest.MakeRouter(rest.Get("/r/:id", NewImageMeta(mock).Get))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("GET", "http://1.2.3.4/r/test", nil))

	recorded.CodeIs(http.StatusNotFound)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(controllers.ErrNotFound))
}

func TestImageMetaGet(t *testing.T) {

	img := images.NewImageMetaFromPublic(nil)
	now := time.Now()
	img.LastUpdated = now
	j, _ := json.Marshal(img)

	mock := &ImageControllerMock{
		mockGet: func(user users.UserI, id string) (*images.ImageMeta, error) {
			return img, nil
		},
	}

	router, err := rest.MakeRouter(rest.Get("/r/:id", NewImageMeta(mock).Get))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("GET", "http://1.2.3.4/r/test", nil))

	recorded.CodeIs(http.StatusOK)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(string(j))
	recorded.HeaderIs("Last-Modified", now.UTC().Format(http.TimeFormat))
}

func TestImageMetaCreate(t *testing.T) {

	id := "123-dummy-id-123"
	now := time.Now()

	img := images.NewImageMetaFromPublic(images.NewImageMetaPublic("My name"))
	img.LastUpdated = now
	img.Id = id
	j, _ := json.Marshal(img)

	mock := &ImageControllerMock{
		mockCreate: func(user users.UserI, public *images.ImageMetaPublic) (*images.ImageMeta, error) {
			i := images.NewImageMetaFromPublic(public)
			i.LastUpdated = now
			i.Id = id

			return i, nil
		},
	}

	router, err := rest.MakeRouter(rest.Post("/r/", NewImageMeta(mock).Create))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("POST", "http://1.2.3.4/r/", img))

	recorded.CodeIs(http.StatusCreated)
	recorded.ContentTypeIsJson()
	recorded.HeaderIs("Location", "/api/0.0.1/images/"+id)
	recorded.BodyIs(string(j))
}

func TestImageMetaCreateInvalidAttr(t *testing.T) {

	img := images.NewImageMetaFromPublic(images.NewImageMetaPublic(""))

	mock := &ImageControllerMock{
		mockCreate: func(user users.UserI, public *images.ImageMetaPublic) (*images.ImageMeta, error) {
			return images.NewImageMetaFromPublic(public), nil
		},
	}

	router, err := rest.MakeRouter(rest.Post("/r/", NewImageMeta(mock).Create))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("POST", "http://1.2.3.4/r/", img))

	recorded.CodeIs(http.StatusBadRequest)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(images.ErrMissingImageAttrName))
}

func TestImageMetaCreateInvalidPayload(t *testing.T) {

	mock := &ImageControllerMock{
		mockCreate: func(user users.UserI, public *images.ImageMetaPublic) (*images.ImageMeta, error) {
			return images.NewImageMetaFromPublic(public), nil
		},
	}

	router, err := rest.MakeRouter(rest.Post("/r/", NewImageMeta(mock).Create))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("POST", "http://1.2.3.4/r/", "ala ma kota"))

	recorded.CodeIs(http.StatusBadRequest)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(
		errors.New("json: cannot unmarshal string into Go value of type images.ImageMetaPublic")))
}

func TestImageMetaCreateControlerError(t *testing.T) {

	apiError := errors.New("Dummy error")
	img := images.NewImageMetaFromPublic(images.NewImageMetaPublic("Name"))

	mock := &ImageControllerMock{
		mockCreate: func(user users.UserI, public *images.ImageMetaPublic) (*images.ImageMeta, error) {
			return nil, apiError
		},
	}

	router, err := rest.MakeRouter(rest.Post("/r/", NewImageMeta(mock).Create))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("POST", "http://1.2.3.4/r/", img))

	recorded.CodeIs(http.StatusInternalServerError)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(apiError))
}
<commit_msg>Convert ImageMetaLookup tests to table tests.<commit_after>package handlers

import (
	"encoding/json"
	"errors"
	"net/http"
	"runtime"
	"testing"
	"time"

	"github.com/ant0ine/go-json-rest/rest"
	"github.com/ant0ine/go-json-rest/rest/test"
	"github.com/mendersoftware/artifacts/controllers"
	"github.com/mendersoftware/artifacts/models/images"
	"github.com/mendersoftware/artifacts/models/users"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

type SetUserMiddleware struct{}

func (mw *SetUserMiddleware) MiddlewareFunc(handler rest.HandlerFunc) rest.HandlerFunc {
	return func(writer rest.ResponseWriter, request *rest.Request) {
		request.Env["REMOTE_USER"] = "admin"
		handler(writer, request)
	}
}

func TestImageMetaLookup(t *testing.T) {

	testList := []struct {
		responseCode int
		body         string
		results      []*images.ImageMeta
		err          error
	}{
		{http.StatusOK, "[]", make([]*images.ImageMeta, 0), nil},
		{http.StatusInternalServerError, RestErrorMsg(errors.New("My dummy error")), nil, errors.New("My dummy error")},
	}

	for _, testCase := range testList {
		mock := &ImageControllerMock{
			mockLookup: func(user users.UserI) ([]*images.ImageMeta, error) {
				return testCase.results, testCase.err
			},
		}

		router, err := rest.MakeRouter(rest.Get("/r", NewImageMeta(mock).Lookup))
		if err != nil {
			t.FailNow()
		}

		api := rest.NewApi()
		api.Use(&SetUserMiddleware{})
		api.SetApp(router)

		recorded := test.RunRequest(t, api.MakeHandler(),
			test.MakeSimpleRequest("GET", "http://1.2.3.4/r", nil))

		recorded.CodeIs(testCase.responseCode)
		recorded.ContentTypeIsJson()
		recorded.BodyIs(testCase.body)
	}
}

func TestImageMetaGetNotFound(t *testing.T) {

	mock := &ImageControllerMock{
		mockGet: func(user users.UserI, id string) (*images.ImageMeta, error) {
			return nil, controllers.ErrNotFound
		},
	}

	router, err := rest.MakeRouter(rest.Get("/r/:id", NewImageMeta(mock).Get))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("GET", "http://1.2.3.4/r/test", nil))

	recorded.CodeIs(http.StatusNotFound)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(controllers.ErrNotFound))
}

func TestImageMetaGet(t *testing.T) {

	img := images.NewImageMetaFromPublic(nil)
	now := time.Now()
	img.LastUpdated = now
	j, _ := json.Marshal(img)

	mock := &ImageControllerMock{
		mockGet: func(user users.UserI, id string) (*images.ImageMeta, error) {
			return img, nil
		},
	}

	router, err := rest.MakeRouter(rest.Get("/r/:id", NewImageMeta(mock).Get))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("GET", "http://1.2.3.4/r/test", nil))

	recorded.CodeIs(http.StatusOK)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(string(j))
	recorded.HeaderIs("Last-Modified", now.UTC().Format(http.TimeFormat))
}

func TestImageMetaCreate(t *testing.T) {

	id := "123-dummy-id-123"
	now := time.Now()

	img := images.NewImageMetaFromPublic(images.NewImageMetaPublic("My name"))
	img.LastUpdated = now
	img.Id = id
	j, _ := json.Marshal(img)

	mock := &ImageControllerMock{
		mockCreate: func(user users.UserI, public *images.ImageMetaPublic) (*images.ImageMeta, error) {
			i := images.NewImageMetaFromPublic(public)
			i.LastUpdated = now
			i.Id = id

			return i, nil
		},
	}

	router, err := rest.MakeRouter(rest.Post("/r/", NewImageMeta(mock).Create))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("POST", "http://1.2.3.4/r/", img))

	recorded.CodeIs(http.StatusCreated)
	recorded.ContentTypeIsJson()
	recorded.HeaderIs("Location", "/api/0.0.1/images/"+id)
	recorded.BodyIs(string(j))
}

func TestImageMetaCreateInvalidAttr(t *testing.T) {

	img := images.NewImageMetaFromPublic(images.NewImageMetaPublic(""))

	mock := &ImageControllerMock{
		mockCreate: func(user users.UserI, public *images.ImageMetaPublic) (*images.ImageMeta, error) {
			return images.NewImageMetaFromPublic(public), nil
		},
	}

	router, err := rest.MakeRouter(rest.Post("/r/", NewImageMeta(mock).Create))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("POST", "http://1.2.3.4/r/", img))

	recorded.CodeIs(http.StatusBadRequest)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(images.ErrMissingImageAttrName))
}

func TestImageMetaCreateInvalidPayload(t *testing.T) {

	mock := &ImageControllerMock{
		mockCreate: func(user users.UserI, public *images.ImageMetaPublic) (*images.ImageMeta, error) {
			return images.NewImageMetaFromPublic(public), nil
		},
	}

	router, err := rest.MakeRouter(rest.Post("/r/", NewImageMeta(mock).Create))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("POST", "http://1.2.3.4/r/", "ala ma kota"))

	recorded.CodeIs(http.StatusBadRequest)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(
		errors.New("json: cannot unmarshal string into Go value of type images.ImageMetaPublic")))
}

func TestImageMetaCreateControlerError(t *testing.T) {

	apiError := errors.New("Dummy error")
	img := images.NewImageMetaFromPublic(images.NewImageMetaPublic("Name"))

	mock := &ImageControllerMock{
		mockCreate: func(user users.UserI, public *images.ImageMetaPublic) (*images.ImageMeta, error) {
			return nil, apiError
		},
	}

	router, err := rest.MakeRouter(rest.Post("/r/", NewImageMeta(mock).Create))
	if err != nil {
		t.FailNow()
	}

	api := rest.NewApi()
	api.Use(&SetUserMiddleware{})
	api.SetApp(router)

	recorded := test.RunRequest(t, api.MakeHandler(),
		test.MakeSimpleRequest("POST", "http://1.2.3.4/r/", img))

	recorded.CodeIs(http.StatusInternalServerError)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(RestErrorMsg(apiError))
}
<|endoftext|>


<commit_before>package handlers

import (
	"fmt"
	"github.com/laktek/Stack-on-Go/stackongo"
	"strconv"
)

const MAX_ANSWERS = 4

//searchStackOverflow performs StackOverflow.com search via Stack-On-Go library.
func searchStackOverflow(searchTerm string, numOfResults int) []*SearchResult {
	result := make([]*SearchResult, 0, numOfResults)

	params := make(stackongo.Params)
	params.Add("filter", "withbody")

	session := stackongo.NewSession("stackoverflow")
	questions, err := session.Search(searchTerm, params)
	if err != nil {
		fmt.Printf("Error while retrieving questions:%s\n", err.Error())
		return result
	}

	for i, question := range questions.Items {
		if i == numOfResults {
			break
		}

		sr := SearchResult{}
		sr.Url = question.Link
		sr.Title = question.Title
		sr.Contents = question.Body

		if question.Answer_count > 0 {

			sr.Extras = make(map[string]string)
			sr.ExtrasOrder = make([]string, 0, MAX_ANSWERS)

			// sort answers by score
			params.Sort("votes")

			// is answered?
			answersCount := 0
			if question.Is_answered {
				answer, err := session.GetAnswers([]int{question.Accepted_answer_id}, params)
				if err != nil {
					fmt.Printf("Cannot retrieve a correct answer:%s", err.Error())
				}

				vote := "(vote:" + strconv.Itoa(answer.Items[0].Score) + ") "
				extra := "Answer 1(\u2713):"
				sr.Extras[extra] = vote + answer.Items[0].Body
				sr.ExtrasOrder = append(sr.ExtrasOrder, extra)

				answersCount++
			}

			answers, _ := session.AnswersForQuestions([]int{question.Question_id}, params)
			for _, answer := range answers.Items {
				if answersCount > MAX_ANSWERS {
					break
				}

				if answer.Is_accepted {
					continue
				}

				vote := "(vote:" + strconv.Itoa(answer.Score) + ") "
				extra := "Answer " + strconv.Itoa(answersCount+1) + ":"
				sr.Extras["Answer "+strconv.Itoa(answersCount+1)+":"] = vote + answer.Body
				sr.ExtrasOrder = append(sr.ExtrasOrder, extra)

				answersCount++
			}
		}

		result = append(result, &sr)
	}

	return result
}
<commit_msg>Fix: so handler - prevent index out of range in the case of no accepted answer.<commit_after>package handlers

import (
	"fmt"
	"github.com/laktek/Stack-on-Go/stackongo"
	"strconv"
)

const MAX_ANSWERS = 4

//searchStackOverflow performs StackOverflow.com search via Stack-On-Go library.
func searchStackOverflow(searchTerm string, numOfResults int) []*SearchResult {
	result := make([]*SearchResult, 0, numOfResults)

	params := make(stackongo.Params)
	params.Add("filter", "withbody")

	session := stackongo.NewSession("stackoverflow")
	questions, err := session.Search(searchTerm, params)
	if err != nil {
		fmt.Printf("Error while retrieving questions:%s\n", err.Error())
		return result
	}

	for i, question := range questions.Items {
		if i == numOfResults {
			break
		}

		sr := SearchResult{}
		sr.Url = question.Link
		sr.Title = question.Title
		sr.Contents = question.Body

		if question.Answer_count > 0 {

			sr.Extras = make(map[string]string)
			sr.ExtrasOrder = make([]string, 0, MAX_ANSWERS)

			// sort answers by score
			params := make(stackongo.Params)
			params.Add("filter", "withbody")
			params.Sort("votes")

			// is answered?
			answersCount := 0
			if question.Is_answered && question.Accepted_answer_id > 0 {
				answer, err := session.GetAnswers([]int{question.Accepted_answer_id}, params)
				if err != nil {
					fmt.Printf("Cannot retrieve a correct answer:%s", err.Error())
				} else {
					vote := "(vote:" + strconv.Itoa(answer.Items[0].Score) + ") "
					extra := "Answer 1(\u2713):"
					sr.Extras[extra] = vote + answer.Items[0].Body
					sr.ExtrasOrder = append(sr.ExtrasOrder, extra)

					answersCount++
				}
			}

			answers, _ := session.AnswersForQuestions([]int{question.Question_id}, params)
			for _, answer := range answers.Items {
				if answersCount > MAX_ANSWERS {
					break
				}

				if answer.Is_accepted {
					continue
				}

				vote := "(vote:" + strconv.Itoa(answer.Score) + ") "
				extra := "Answer " + strconv.Itoa(answersCount+1) + ":"
				sr.Extras["Answer "+strconv.Itoa(answersCount+1)+":"] = vote + answer.Body
				sr.ExtrasOrder = append(sr.ExtrasOrder, extra)

				answersCount++
			}
		}

		result = append(result, &sr)
	}

	return result
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ifacemonitor

import (
	log "github.com/Sirupsen/logrus"
	"github.com/projectcalico/felix/go/felix/set"
	"github.com/vishvananda/netlink"
	"syscall"
	"time"
)

type State string

const (
	StateUp   = "up"
	StateDown = "down"
)

type InterfaceStateCallback func(ifaceName string, ifaceState State)
type AddrStateCallback func(ifaceName string, addrs []string)

type InterfaceMonitor struct {
	upIfaces     set.Set
	Callback     InterfaceStateCallback
	AddrCallback AddrStateCallback
	ifaceName    map[int]string
	ifaceAddrs   map[int][]string
}

func New() *InterfaceMonitor {
	return &InterfaceMonitor{
		upIfaces:   set.New(),
		ifaceName:  make(map[int]string),
		ifaceAddrs: make(map[int][]string),
	}
}

func (m *InterfaceMonitor) MonitorInterfaces() {
	log.Info("Interface monitoring thread started.")
	updates := make(chan netlink.LinkUpdate)
	addr_updates := make(chan netlink.AddrUpdate)
	cancel := make(chan struct{})

	if err := netlink.LinkSubscribe(updates, cancel); err != nil {
		log.WithError(err).Fatal("Failed to subscribe to link updates")
	}
	if err := netlink.AddrSubscribe(addr_updates, cancel); err != nil {
		log.WithError(err).Fatal("Failed to subscribe to addr updates")
	}
	log.Info("Subscribed to netlink updates.")

	// Start of day, do a resync to notify all our existing interfaces.  We also do periodic
	// resyncs because it's not clear what the ordering guarantees are for our netlink
	// subscription vs a list operation as used by resync().
	err := m.resync()
	if err != nil {
		log.WithError(err).Fatal("Failed to read link states from netlink.")
	}

	// Schedule periodic resyncs after that.
	resyncTicker := time.NewTicker(10 * time.Second)
readLoop:
	for {
		select {
		case update, ok := <-updates:
			if !ok {
				break readLoop
			}
			m.handleNetlinkUpdate(update)
		case addr_update, ok := <-addr_updates:
			if !ok {
				break readLoop
			}
			m.handleNetlinkAddrUpdate(addr_update)
		case <-resyncTicker.C:
			err := m.resync()
			if err != nil {
				log.WithError(err).Fatal("Failed to read link states from netlink.")
			}
		}
	}
	log.Fatal("Failed to read events from Netlink.")
}

func (m *InterfaceMonitor) handleNetlinkUpdate(update netlink.LinkUpdate) {
	attrs := update.Attrs()
	if attrs == nil {
		// Defensive, some sort of interface that the netlink lib doesn't understand?
		log.WithField("update", update).Warn("Missing attributes on netlink update.")
		return
	}
	msgType := update.Header.Type
	ifaceExists := msgType == syscall.RTM_NEWLINK // Alternative is an RTM_DELLINK
	m.storeUpdateAndNotifyOnChange(ifaceExists, attrs)
}

func (m *InterfaceMonitor) handleNetlinkAddrUpdate(update netlink.AddrUpdate) {
	addr := update.LinkAddress.IP.String()
	ifIndex := update.LinkIndex
	exists := update.NewAddr
	log.WithFields(log.Fields{
		"addr":    addr,
		"ifIndex": ifIndex,
		"exists":  exists,
	}).Info("Netlink address update.")

	_, ifaceKnown := m.ifaceName[ifIndex]
	if !ifaceKnown {
		log.WithField("ifIndex", ifIndex).Warn("No known iface with this index.")
		return
	}

	if exists {
		if !m.addrKnownForIface(addr, ifIndex) {
			m.addAddrForIface(addr, ifIndex)
			m.notifyIfaceAddrs(ifIndex)
		}
	} else {
		if m.addrKnownForIface(addr, ifIndex) {
			m.delAddrForIface(addr, ifIndex)
			m.notifyIfaceAddrs(ifIndex)
		}
	}
}

func (m *InterfaceMonitor) addrKnownForIface(addr string, ifIndex int) bool {
	for _, known := range m.ifaceAddrs[ifIndex] {
		if addr == known {
			return true
		}
	}
	return false
}

func (m *InterfaceMonitor) addAddrForIface(addr string, ifIndex int) {
	m.ifaceAddrs[ifIndex] = append(m.ifaceAddrs[ifIndex], addr)
}

func (m *InterfaceMonitor) delAddrForIface(addr string, ifIndex int) {
	for i, known := range m.ifaceAddrs[ifIndex] {
		if addr == known {
			last := len(m.ifaceAddrs[ifIndex]) - 1
			m.ifaceAddrs[ifIndex][i] = m.ifaceAddrs[ifIndex][last]
			m.ifaceAddrs[ifIndex] = m.ifaceAddrs[ifIndex][:last]
			break
		}
	}
}

func (m *InterfaceMonitor) notifyIfaceAddrs(ifIndex int) {
	m.AddrCallback(m.ifaceName[ifIndex], m.ifaceAddrs[ifIndex])
}

func (m *InterfaceMonitor) storeUpdateAndNotifyOnChange(ifaceExists bool, attrs *netlink.LinkAttrs) {
	// Store or remove mapping between this interface's index and name.
	if ifaceExists {
		m.ifaceName[attrs.Index] = attrs.Name
	} else {
		delete(m.ifaceName, attrs.Index)
	}
	// We need the operstate of the interface; this is carried in the IFF_RUNNING flag.
	// The IFF_UP flag contains the admin state, which doesn't tell us whether we can
	// program routes etc.
	rawFlags := attrs.RawFlags
	ifaceIsUp := ifaceExists && rawFlags&syscall.IFF_RUNNING != 0
	ifaceName := attrs.Name
	ifaceWasUp := m.upIfaces.Contains(ifaceName)
	logCxt := log.WithField("ifaceName", ifaceName)
	if ifaceIsUp && !ifaceWasUp {
		logCxt.Debug("Interface now up")
		m.upIfaces.Add(ifaceName)
		m.Callback(ifaceName, StateUp)
	} else if ifaceWasUp && !ifaceIsUp {
		logCxt.Debug("Interface now down")
		m.upIfaces.Discard(ifaceName)
		m.Callback(ifaceName, StateDown)
	}
}

func (m *InterfaceMonitor) resync() error {
	log.Debug("Resyncing interface state.")
	links, err := netlink.LinkList()
	if err != nil {
		log.WithError(err).Warn("Netlink list operation failed.")
		return err
	}
	currentIfaces := set.New()
	for _, link := range links {
		attrs := link.Attrs()
		if attrs == nil {
			// Defensive, some sort of interface that the netlink lib doesn't
			// understand?
			log.WithField("link", link).Warn("Missing attributes on netlink update.")
			continue
		}
		currentIfaces.Add(attrs.Name)
		m.storeUpdateAndNotifyOnChange(true, attrs)

		ifIndex := attrs.Index
		new_addrs := []string{}
		for _, family := range [2]int{netlink.FAMILY_V4, netlink.FAMILY_V6} {
			addrs, err := netlink.AddrList(link, family)
			if err != nil {
				log.WithError(err).Warn("Netlink addr list operation failed.")
				return err
			}
			for _, addr := range addrs {
				new_addrs = append(new_addrs, addr.IPNet.IP.String())
			}
		}
		m.ifaceAddrs[ifIndex] = new_addrs
		m.notifyIfaceAddrs(ifIndex)
	}
	m.upIfaces.Iter(func(name interface{}) error {
		if currentIfaces.Contains(name) {
			return nil
		}
		log.WithField("ifaceName", name).Info("Spotted interface removal on resync.")
		return set.RemoveItem
	})
	return nil
}
<commit_msg>Code review markups<commit_after>// Copyright (c) 2016 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ifacemonitor

import (
	log "github.com/Sirupsen/logrus"
	"github.com/projectcalico/felix/go/felix/set"
	"github.com/vishvananda/netlink"
	"syscall"
	"time"
)

type State string

const (
	StateUp   = "up"
	StateDown = "down"
)

type InterfaceStateCallback func(ifaceName string, ifaceState State)
type AddrStateCallback func(ifaceName string, addrs []string)

type InterfaceMonitor struct {
	upIfaces     set.Set
	Callback     InterfaceStateCallback
	AddrCallback AddrStateCallback
	ifaceName    map[int]string
	ifaceAddrs   map[int][]string
}

func New() *InterfaceMonitor {
	return &InterfaceMonitor{
		upIfaces:   set.New(),
		ifaceName:  make(map[int]string),
		ifaceAddrs: make(map[int][]string),
	}
}

func (m *InterfaceMonitor) MonitorInterfaces() {
	log.Info("Interface monitoring thread started.")
	updates := make(chan netlink.LinkUpdate)
	addrUpdates := make(chan netlink.AddrUpdate)
	cancel := make(chan struct{})

	if err := netlink.LinkSubscribe(updates, cancel); err != nil {
		log.WithError(err).Fatal("Failed to subscribe to link updates")
	}
	if err := netlink.AddrSubscribe(addrUpdates, cancel); err != nil {
		log.WithError(err).Fatal("Failed to subscribe to addr updates")
	}
	log.Info("Subscribed to netlink updates.")

	// Start of day, do a resync to notify all our existing interfaces.  We also do periodic
	// resyncs because it's not clear what the ordering guarantees are for our netlink
	// subscription vs a list operation as used by resync().
	err := m.resync()
	if err != nil {
		log.WithError(err).Fatal("Failed to read link states from netlink.")
	}

	// Schedule periodic resyncs after that.
	resyncTicker := time.NewTicker(10 * time.Second)
readLoop:
	for {
		select {
		case update, ok := <-updates:
			if !ok {
				break readLoop
			}
			m.handleNetlinkUpdate(update)
		case addr_update, ok := <-addrUpdates:
			if !ok {
				break readLoop
			}
			m.handleNetlinkAddrUpdate(addr_update)
		case <-resyncTicker.C:
			err := m.resync()
			if err != nil {
				log.WithError(err).Fatal("Failed to read link states from netlink.")
			}
		}
	}
	log.Fatal("Failed to read events from Netlink.")
}

func (m *InterfaceMonitor) handleNetlinkUpdate(update netlink.LinkUpdate) {
	attrs := update.Attrs()
	if attrs == nil {
		// Defensive, some sort of interface that the netlink lib doesn't understand?
		log.WithField("update", update).Warn("Missing attributes on netlink update.")
		return
	}
	msgType := update.Header.Type
	ifaceExists := msgType == syscall.RTM_NEWLINK // Alternative is an RTM_DELLINK
	m.storeUpdateAndNotifyOnChange(ifaceExists, attrs)
}

func (m *InterfaceMonitor) handleNetlinkAddrUpdate(update netlink.AddrUpdate) {
	addr := update.LinkAddress.IP.String()
	ifIndex := update.LinkIndex
	exists := update.NewAddr
	log.WithFields(log.Fields{
		"addr":    addr,
		"ifIndex": ifIndex,
		"exists":  exists,
	}).Info("Netlink address update.")

	_, ifaceKnown := m.ifaceName[ifIndex]
	if !ifaceKnown {
		log.WithField("ifIndex", ifIndex).Warn("No known iface with this index.")
		return
	}

	if exists {
		if !m.addrKnownForIface(addr, ifIndex) {
			m.addAddrForIface(addr, ifIndex)
			m.notifyIfaceAddrs(ifIndex)
		}
	} else {
		if m.addrKnownForIface(addr, ifIndex) {
			m.delAddrForIface(addr, ifIndex)
			m.notifyIfaceAddrs(ifIndex)
		}
	}
}

func (m *InterfaceMonitor) addrKnownForIface(addr string, ifIndex int) bool {
	for _, known := range m.ifaceAddrs[ifIndex] {
		if addr == known {
			return true
		}
	}
	return false
}

func (m *InterfaceMonitor) addAddrForIface(addr string, ifIndex int) {
	m.ifaceAddrs[ifIndex] = append(m.ifaceAddrs[ifIndex], addr)
}

func (m *InterfaceMonitor) delAddrForIface(addr string, ifIndex int) {
	for i, known := range m.ifaceAddrs[ifIndex] {
		if addr == known {
			last := len(m.ifaceAddrs[ifIndex]) - 1
			m.ifaceAddrs[ifIndex][i] = m.ifaceAddrs[ifIndex][last]
			m.ifaceAddrs[ifIndex] = m.ifaceAddrs[ifIndex][:last]
			break
		}
	}
}

func (m *InterfaceMonitor) notifyIfaceAddrs(ifIndex int) {
	m.AddrCallback(m.ifaceName[ifIndex], m.ifaceAddrs[ifIndex])
}

func (m *InterfaceMonitor) storeUpdateAndNotifyOnChange(ifaceExists bool, attrs *netlink.LinkAttrs) {
	// Store or remove mapping between this interface's index and name.
	if ifaceExists {
		m.ifaceName[attrs.Index] = attrs.Name
	} else {
		delete(m.ifaceName, attrs.Index)
	}
	// We need the operstate of the interface; this is carried in the IFF_RUNNING flag.
	// The IFF_UP flag contains the admin state, which doesn't tell us whether we can
	// program routes etc.
	rawFlags := attrs.RawFlags
	ifaceIsUp := ifaceExists && rawFlags&syscall.IFF_RUNNING != 0
	ifaceName := attrs.Name
	ifaceWasUp := m.upIfaces.Contains(ifaceName)
	logCxt := log.WithField("ifaceName", ifaceName)
	if ifaceIsUp && !ifaceWasUp {
		logCxt.Debug("Interface now up")
		m.upIfaces.Add(ifaceName)
		m.Callback(ifaceName, StateUp)
	} else if ifaceWasUp && !ifaceIsUp {
		logCxt.Debug("Interface now down")
		m.upIfaces.Discard(ifaceName)
		m.Callback(ifaceName, StateDown)
	}
}

func (m *InterfaceMonitor) resync() error {
	log.Debug("Resyncing interface state.")
	links, err := netlink.LinkList()
	if err != nil {
		log.WithError(err).Warn("Netlink list operation failed.")
		return err
	}
	currentIfaces := set.New()
	for _, link := range links {
		attrs := link.Attrs()
		if attrs == nil {
			// Defensive, some sort of interface that the netlink lib doesn't
			// understand?
			log.WithField("link", link).Warn("Missing attributes on netlink update.")
			continue
		}
		currentIfaces.Add(attrs.Name)
		m.storeUpdateAndNotifyOnChange(true, attrs)

		ifIndex := attrs.Index
		new_addrs := []string{}
		for _, family := range [2]int{netlink.FAMILY_V4, netlink.FAMILY_V6} {
			addrs, err := netlink.AddrList(link, family)
			if err != nil {
				log.WithError(err).Warn("Netlink addr list operation failed.")
				return err
			}
			for _, addr := range addrs {
				new_addrs = append(new_addrs, addr.IPNet.IP.String())
			}
		}
		m.ifaceAddrs[ifIndex] = new_addrs
		m.notifyIfaceAddrs(ifIndex)
	}
	m.upIfaces.Iter(func(name interface{}) error {
		if currentIfaces.Contains(name) {
			return nil
		}
		log.WithField("ifaceName", name).Info("Spotted interface removal on resync.")
		return set.RemoveItem
	})
	return nil
}
<|endoftext|>


<commit_before>package neo4j

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"koding/tools/config"
	"labix.org/v2/mgo/bson"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var (
	BASE_URL         = config.Current.Neo4j.Write + ":" + strconv.Itoa(config.Current.Neo4j.Port)
	INDEX_NODE_PATH  = "/db/data/index/node/koding"
	UNIQUE_NODE_PATH = "/db/data/index/node/koding?unique"
	INDEX_PATH       = "/db/data/index/node"
	NODE_URL         = "/db/data/node"
	MAX_RETRIES      = 5
	TIMEOUT          = 3
	DEADLINE         = 30
	CYPHER_PATH      = "db/data/cypher"
	CYPHER_URL       = fmt.Sprintf("%v/%v", BASE_URL, CYPHER_PATH)
)

type Relationship struct {
	Id         bson.ObjectId `bson:"_id,omitempty"`
	TargetId   bson.ObjectId `bson:"targetId,omitempty"`
	TargetName string        `bson:"targetName"`
	SourceId   bson.ObjectId `bson:"sourceId,omitempty"`
	SourceName string        `bson:"sourceName"`
	As         string        `bson:"as"`
	Timestamp  time.Time     `bson:"timestamp"`
	Data       bson.Binary
}

// Setup the dial timeout
func dialTimeout(timeout time.Duration, deadline time.Duration) func(network, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, timeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(time.Now().Add(deadline))
		return conn, nil
	}
}

// Gets URL and string data to be sent and makes POST request
// reads response body and returns as string
func sendRequest(requestType, url, data string, attempt int) string {

	// Set the timeout & deadline
	timeOut := time.Duration(TIMEOUT) * time.Second
	deadLine := time.Duration(DEADLINE) * time.Second

	transport := http.Transport{
		Dial: dialTimeout(timeOut, deadLine),
	}

	client := http.Client{
		Transport: &transport,
	}

	//convert string into bytestream
	dataByte := strings.NewReader(data)
	req, err := http.NewRequest(requestType, url, dataByte)

	// read response body
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	res, err := client.Do(req)
	if err != nil && attempt <= MAX_RETRIES {
		fmt.Print(err)
		attempt++
		sendRequest(requestType, url, data, attempt)
	}
	if err != nil && attempt > MAX_RETRIES {
		panic(fmt.Sprintf("req to %v timed out after %v retries", url, attempt))
	}

	body, _ := ioutil.ReadAll(res.Body)

	defer res.Body.Close()

	return string(body)

}

// connect source and target with relation property
// response will be object
func CreateRelationship(relation, source, target string) map[string]interface{} {

	relationshipData := fmt.Sprintf(`{"to" : "%s", "type" : "%s" }`, target, relation)
	relRes := sendRequest("POST", fmt.Sprintf("%s", source), relationshipData, 1)

	relNode, err := jsonDecode(relRes)
	if err != nil {
		fmt.Println("Problem with relation response", relRes)
	}

	return relNode
}

// connect source and target with relation property
// response will be object
func CreateRelationshipWithData(relation, source, target, data string) map[string]interface{} {

	relationshipData := fmt.Sprintf(`{"to" : "%s", "type" : "%s", "data" : %s }`, target, relation, data)
	relRes := sendRequest("POST", fmt.Sprintf("%s", source), relationshipData, 1)

	relNode, err := jsonDecode(relRes)
	if err != nil {
		fmt.Println("Problem with relation response", relRes)
	}

	return relNode
}

// creates a unique node with given id and node name
// response will be Object
func CreateUniqueNode(id string, name string) map[string]interface{} {

	url := BASE_URL + UNIQUE_NODE_PATH

	postData := generatePostJsonData(id, name)

	response := sendRequest("POST", url, postData, 1)

	node, err := jsonDecode(response)
	if err != nil {
		fmt.Println("Problem with unique node creation response", response)
	}

	return node
}

// deletes a relation between two node using relationship info
func DeleteRelationship(sourceId, targetId, relationship string) bool {

	//get source node information
	sourceInfo := GetNode(sourceId)

	//get target node information
	targetInfo := GetNode(targetId)

	if len(sourceInfo) < 1 || len(targetInfo) < 1 {
		return false
	}

	if _, ok := sourceInfo[0]["self"]; !ok {
		return false
	}

	if _, ok := targetInfo[0]["self"]; !ok {
		return false
	}

	// create  url to get relationship information of source node
	relationshipsURL := fmt.Sprintf("%s", sourceInfo[0]["self"]) + "/relationships/all/" + relationship

	//this request returns objects in an array
	response := sendRequest("GET", relationshipsURL, "", 1)
	//so use json array decoder
	relationships, err := jsonArrayDecode(response)
	if err != nil {
		fmt.Println("Problem with unique node creation response", response)
		return false
	}

	if len(relationships) < 1 {
		return false
	}

	if _, ok := relationships[0]["self"]; !ok {
		return false
	}

	foundNode := false

	for _, relation := range relationships {
		if relation["end"] == targetInfo[0]["self"] {
			toBeDeletedRelationURL := fmt.Sprintf("%s", relation["self"])
			sendRequest("DELETE", toBeDeletedRelationURL, "", 1)
			foundNode = true

			break
		}
	}

	if !foundNode {
		fmt.Println("not found!", relationships[0]["self"])
	}

	return true
}

// gets node from neo4j with given unique node id
//response will be object
func GetNode(id string) []map[string]interface{} {

	url := BASE_URL + INDEX_NODE_PATH + "/id/" + id

	response := sendRequest("GET", url, "", 1)

	nodeData, err := jsonArrayDecode(response)
	if err != nil {
		fmt.Println("Problem with response", response)
	}

	return nodeData
}

// updates node with given data
// response will be object
func UpdateNode(id, propertiesJSON string) map[string]interface{} {

	node := GetNode(id)

	if len(node) < 1 {
		return nil
	}

	//if self is not there!
	if _, ok := node[0]["self"]; !ok {
		return nil
	}

	// create  url to get relationship information of source node
	propertiesURL := fmt.Sprintf("%s", node[0]["self"]) + "/properties"

	response := sendRequest("PUT", propertiesURL, propertiesJSON, 1)
	if response != "" {
		res, err := jsonDecode(response)
		if err != nil {
			fmt.Println("Problem with response", err, res)
		}
	}

	return make(map[string]interface{})
}

func DeleteNode(id string) bool {

	node := GetNode(id)

	if len(node) < 1 {
		return false
	}

	//if self is not there!
	selfUrl, ok := node[0]["self"]
	if !ok {
		return false
	}

	splitStrings := strings.Split(selfUrl.(string), "/")
	nodeId := splitStrings[len(splitStrings)-1]

	query := fmt.Sprintf(`
    {"query" : "START n=node(%v) MATCH n-[r?]-items DELETE r, n"}
  `, nodeId)

	response := sendRequest("POST", CYPHER_URL, query, 1)

	var result map[string][]interface{}
	err := json.Unmarshal([]byte(response), &result)
	if err != nil {
		fmt.Println("Deleting node Marshalling error:", err)
		return false
	}

	return true
}

// creates a unique tree head node to hold all nodes
// it is called once during runtime while initializing
func CreateUniqueIndex(name string) {
	//create unique index
	url := BASE_URL + INDEX_PATH

	bd := sendRequest("POST", url, `{"name":"`+name+`"}`, 1)

	fmt.Println("Created unique index for data", bd)
}

// This is a custom json string generator as http request body to neo4j
func generatePostJsonData(id, name string) string {
	return fmt.Sprintf(`{ "key" : "id", "value" : "%s", "properties" : { "id" : "%s", "name" : "%s" } }`, id, id, name)
}

//here, mapping of decoded json
func jsonArrayDecode(data string) ([]map[string]interface{}, error) {
	var source []map[string]interface{}

	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		fmt.Println("Marshalling error:", err)
		return nil, err
	}

	return source, nil
}

//here, mapping of decoded json
func jsonDecode(data string) (map[string]interface{}, error) {
	var source map[string]interface{}

	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		fmt.Println("Marshalling error:", err)
		return nil, err
	}

	return source, nil
}

var NotAllowedNames = []string{
	"CStatusActivity",
	"CFolloweeBucketActivity",
	"CFollowerBucketActivity",
	"CCodeSnipActivity",
	"CDiscussionActivity",
	"CReplieeBucketActivity",
	"CReplierBucketActivity",
	"CBlogPostActivity",
	"CNewMemberBucketActivity",
	"CTutorialActivity",
	"CLikeeBucketActivity",
	"CLikerBucketActivity",
	"CInstalleeBucketActivity",
	"CInstallerBucketActivity",
	"CActivity",
	"CRunnableActivity",
	"JAppStorage",
	"JFeed",
}
<commit_msg>neo4j: increase timeout<commit_after>package neo4j

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"koding/tools/config"
	"labix.org/v2/mgo/bson"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var (
	BASE_URL         = config.Current.Neo4j.Write + ":" + strconv.Itoa(config.Current.Neo4j.Port)
	INDEX_NODE_PATH  = "/db/data/index/node/koding"
	UNIQUE_NODE_PATH = "/db/data/index/node/koding?unique"
	INDEX_PATH       = "/db/data/index/node"
	NODE_URL         = "/db/data/node"
	MAX_RETRIES      = 5
	TIMEOUT          = 20
	DEADLINE         = 40
	CYPHER_PATH      = "db/data/cypher"
	CYPHER_URL       = fmt.Sprintf("%v/%v", BASE_URL, CYPHER_PATH)
)

type Relationship struct {
	Id         bson.ObjectId `bson:"_id,omitempty"`
	TargetId   bson.ObjectId `bson:"targetId,omitempty"`
	TargetName string        `bson:"targetName"`
	SourceId   bson.ObjectId `bson:"sourceId,omitempty"`
	SourceName string        `bson:"sourceName"`
	As         string        `bson:"as"`
	Timestamp  time.Time     `bson:"timestamp"`
	Data       bson.Binary
}

// Setup the dial timeout
func dialTimeout(timeout time.Duration, deadline time.Duration) func(network, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, timeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(time.Now().Add(deadline))
		return conn, nil
	}
}

// Gets URL and string data to be sent and makes POST request
// reads response body and returns as string
func sendRequest(requestType, url, data string, attempt int) string {

	// Set the timeout & deadline
	timeOut := time.Duration(TIMEOUT) * time.Second
	deadLine := time.Duration(DEADLINE) * time.Second

	transport := http.Transport{
		Dial: dialTimeout(timeOut, deadLine),
	}

	client := http.Client{
		Transport: &transport,
	}

	//convert string into bytestream
	dataByte := strings.NewReader(data)
	req, err := http.NewRequest(requestType, url, dataByte)

	// read response body
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	res, err := client.Do(req)
	if err != nil && attempt <= MAX_RETRIES {
		fmt.Print(err)
		attempt++
		sendRequest(requestType, url, data, attempt)
	}
	if err != nil && attempt > MAX_RETRIES {
		panic(fmt.Sprintf("req to %v timed out after %v retries", url, attempt))
	}

	body, _ := ioutil.ReadAll(res.Body)

	defer res.Body.Close()

	return string(body)

}

// connect source and target with relation property
// response will be object
func CreateRelationship(relation, source, target string) map[string]interface{} {

	relationshipData := fmt.Sprintf(`{"to" : "%s", "type" : "%s" }`, target, relation)
	relRes := sendRequest("POST", fmt.Sprintf("%s", source), relationshipData, 1)

	relNode, err := jsonDecode(relRes)
	if err != nil {
		fmt.Println("Problem with relation response", relRes)
	}

	return relNode
}

// connect source and target with relation property
// response will be object
func CreateRelationshipWithData(relation, source, target, data string) map[string]interface{} {

	relationshipData := fmt.Sprintf(`{"to" : "%s", "type" : "%s", "data" : %s }`, target, relation, data)
	relRes := sendRequest("POST", fmt.Sprintf("%s", source), relationshipData, 1)

	relNode, err := jsonDecode(relRes)
	if err != nil {
		fmt.Println("Problem with relation response", relRes)
	}

	return relNode
}

// creates a unique node with given id and node name
// response will be Object
func CreateUniqueNode(id string, name string) map[string]interface{} {

	url := BASE_URL + UNIQUE_NODE_PATH

	postData := generatePostJsonData(id, name)

	response := sendRequest("POST", url, postData, 1)

	node, err := jsonDecode(response)
	if err != nil {
		fmt.Println("Problem with unique node creation response", response)
	}

	return node
}

// deletes a relation between two node using relationship info
func DeleteRelationship(sourceId, targetId, relationship string) bool {

	//get source node information
	sourceInfo := GetNode(sourceId)

	//get target node information
	targetInfo := GetNode(targetId)

	if len(sourceInfo) < 1 || len(targetInfo) < 1 {
		return false
	}

	if _, ok := sourceInfo[0]["self"]; !ok {
		return false
	}

	if _, ok := targetInfo[0]["self"]; !ok {
		return false
	}

	// create  url to get relationship information of source node
	relationshipsURL := fmt.Sprintf("%s", sourceInfo[0]["self"]) + "/relationships/all/" + relationship

	//this request returns objects in an array
	response := sendRequest("GET", relationshipsURL, "", 1)
	//so use json array decoder
	relationships, err := jsonArrayDecode(response)
	if err != nil {
		fmt.Println("Problem with unique node creation response", response)
		return false
	}

	if len(relationships) < 1 {
		return false
	}

	if _, ok := relationships[0]["self"]; !ok {
		return false
	}

	foundNode := false

	for _, relation := range relationships {
		if relation["end"] == targetInfo[0]["self"] {
			toBeDeletedRelationURL := fmt.Sprintf("%s", relation["self"])
			sendRequest("DELETE", toBeDeletedRelationURL, "", 1)
			foundNode = true

			break
		}
	}

	if !foundNode {
		fmt.Println("not found!", relationships[0]["self"])
	}

	return true
}

// gets node from neo4j with given unique node id
//response will be object
func GetNode(id string) []map[string]interface{} {

	url := BASE_URL + INDEX_NODE_PATH + "/id/" + id

	response := sendRequest("GET", url, "", 1)

	nodeData, err := jsonArrayDecode(response)
	if err != nil {
		fmt.Println("Problem with response", response)
	}

	return nodeData
}

// updates node with given data
// response will be object
func UpdateNode(id, propertiesJSON string) map[string]interface{} {

	node := GetNode(id)

	if len(node) < 1 {
		return nil
	}

	//if self is not there!
	if _, ok := node[0]["self"]; !ok {
		return nil
	}

	// create  url to get relationship information of source node
	propertiesURL := fmt.Sprintf("%s", node[0]["self"]) + "/properties"

	response := sendRequest("PUT", propertiesURL, propertiesJSON, 1)
	if response != "" {
		res, err := jsonDecode(response)
		if err != nil {
			fmt.Println("Problem with response", err, res)
		}
	}

	return make(map[string]interface{})
}

func DeleteNode(id string) bool {

	node := GetNode(id)

	if len(node) < 1 {
		return false
	}

	//if self is not there!
	selfUrl, ok := node[0]["self"]
	if !ok {
		return false
	}

	splitStrings := strings.Split(selfUrl.(string), "/")
	nodeId := splitStrings[len(splitStrings)-1]

	query := fmt.Sprintf(`
    {"query" : "START n=node(%v) MATCH n-[r?]-items DELETE r, n"}
  `, nodeId)

	response := sendRequest("POST", CYPHER_URL, query, 1)

	var result map[string][]interface{}
	err := json.Unmarshal([]byte(response), &result)
	if err != nil {
		fmt.Println("Deleting node Marshalling error:", err)
		return false
	}

	return true
}

// creates a unique tree head node to hold all nodes
// it is called once during runtime while initializing
func CreateUniqueIndex(name string) {
	//create unique index
	url := BASE_URL + INDEX_PATH

	bd := sendRequest("POST", url, `{"name":"`+name+`"}`, 1)

	fmt.Println("Created unique index for data", bd)
}

// This is a custom json string generator as http request body to neo4j
func generatePostJsonData(id, name string) string {
	return fmt.Sprintf(`{ "key" : "id", "value" : "%s", "properties" : { "id" : "%s", "name" : "%s" } }`, id, id, name)
}

//here, mapping of decoded json
func jsonArrayDecode(data string) ([]map[string]interface{}, error) {
	var source []map[string]interface{}

	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		fmt.Println("Marshalling error:", err)
		return nil, err
	}

	return source, nil
}

//here, mapping of decoded json
func jsonDecode(data string) (map[string]interface{}, error) {
	var source map[string]interface{}

	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		fmt.Println("Marshalling error:", err)
		return nil, err
	}

	return source, nil
}

var NotAllowedNames = []string{
	"CStatusActivity",
	"CFolloweeBucketActivity",
	"CFollowerBucketActivity",
	"CCodeSnipActivity",
	"CDiscussionActivity",
	"CReplieeBucketActivity",
	"CReplierBucketActivity",
	"CBlogPostActivity",
	"CNewMemberBucketActivity",
	"CTutorialActivity",
	"CLikeeBucketActivity",
	"CLikerBucketActivity",
	"CInstalleeBucketActivity",
	"CInstallerBucketActivity",
	"CActivity",
	"CRunnableActivity",
	"JAppStorage",
	"JFeed",
}
<|endoftext|>