"""
Logic for cleaning and processing zipcode data
"""
import pandas as pd
import string
import zipcodes

# Mapping of zipcode to Municipality for all zipcodes that are uniquely associated with one municipality, calculated in `zipcode_muni_mapping.ipynb`
ZIP_MUNI_MAP = {
    '01001': 'Agawam',
    '01003': 'Amherst',
    '01004': 'Amherst',
    '01005': 'Barre',
    '01007': 'Belchertown',
    '01008': 'Blandford',
    '01009': 'Palmer',
    '01010': 'Brimfield',
    '01011': 'Chester',
    '01012': 'Chesterfield',
    '01013': 'Chicopee',
    '01014': 'Chicopee',
    '01020': 'Chicopee',
    '01021': 'Chicopee',
    '01022': 'Chicopee',
    '01026': 'Cummington',
    '01028': 'East Longmeadow',
    '01029': 'Otis',
    '01030': 'Agawam',
    '01031': 'Hardwick',
    '01032': 'Goshen',
    '01033': 'Granby',
    '01034': 'Granville',
    '01035': 'Hadley',
    '01036': 'Hampden',
    '01037': 'Hardwick',
    '01038': 'Hatfield',
    '01039': 'Williamsburg',
    '01040': 'Holyoke',
    '01041': 'Holyoke',
    '01050': 'Huntington',
    '01053': 'Northampton',
    '01054': 'Leverett',
    '01056': 'Ludlow',
    '01057': 'Monson',
    '01059': 'Amherst',
    '01060': 'Northampton',
    '01061': 'Northampton',
    '01062': 'Northampton',
    '01063': 'Northampton',
    '01066': 'Hatfield',
    '01068': 'Oakham',
    '01069': 'Palmer',
    '01070': 'Plainfield',
    '01071': 'Russell',
    '01072': 'Shutesbury',
    '01073': 'Southampton',
    '01074': 'Barre',
    '01075': 'South Hadley',
    '01077': 'Southwick',
    '01079': 'Palmer',
    '01080': 'Palmer',
    '01081': 'Wales',
    '01082': 'Ware',
    '01083': 'Warren',
    '01084': 'Chesterfield',
    '01086': 'Westfield',
    '01088': 'Hatfield',
    '01089': 'West Springfield',
    '01090': 'West Springfield',
    '01092': 'Warren',
    '01093': 'Whately',
    '01094': 'Hardwick',
    '01095': 'Wilbraham',
    '01096': 'Williamsburg',
    '01097': 'Russell',
    '01098': 'Worthington',
    '01101': 'Springfield',
    '01102': 'Springfield',
    '01103': 'Springfield',
    '01104': 'Springfield',
    '01105': 'Springfield',
    '01106': 'Longmeadow',
    '01107': 'Springfield',
    '01108': 'Springfield',
    '01109': 'Springfield',
    '01111': 'Springfield',
    '01115': 'Springfield',
    '01116': 'Longmeadow',
    '01118': 'Springfield',
    '01119': 'Springfield',
    '01128': 'Springfield',
    '01129': 'Springfield',
    '01133': 'Springfield',
    '01138': 'Springfield',
    '01139': 'Springfield',
    '01144': 'Springfield',
    '01151': 'Springfield',
    '01152': 'Springfield',
    '01195': 'Springfield',
    '01199': 'Springfield',
    '01201': 'Pittsfield',
    '01202': 'Pittsfield',
    '01203': 'Pittsfield',
    '01220': 'Adams',
    '01222': 'Sheffield',
    '01224': 'Lanesborough',
    '01225': 'Cheshire',
    '01227': 'Dalton',
    '01229': 'Stockbridge',
    '01230': 'Great Barrington',
    '01235': 'Hinsdale',
    '01236': 'West Stockbridge',
    '01238': 'Lee',
    '01240': 'Lenox',
    '01242': 'Lenox',
    '01243': 'Middlefield',
    '01244': 'New Marlborough',
    '01245': 'Monterey',
    '01252': 'Egremont',
    '01253': 'Otis',
    '01254': 'Richmond',
    '01255': 'Sandisfield',
    '01256': 'Savoy',
    '01257': 'Sheffield',
    '01259': 'New Marlborough',
    '01260': 'Lee',
    '01262': 'Stockbridge',
    '01263': 'Stockbridge',
    '01264': 'Tyringham',
    '01266': 'West Stockbridge',
    '01267': 'Williamstown',
    '01270': 'Windsor',
    '01301': 'Greenfield',
    '01302': 'Greenfield',
    '01330': 'Ashfield',
    '01338': 'Buckland',
    '01340': 'Colrain',
    '01341': 'Conway',
    '01342': 'Deerfield',
    '01343': 'Florida',
    '01344': 'Erving',
    '01346': 'Heath',
    '01347': 'Montague',
    '01349': 'Montague',
    '01350': 'Monroe',
    '01351': 'Montague',
    '01354': 'Gill',
    '01355': 'New Salem',
    '01360': 'Northfield',
    '01364': 'Orange',
    '01366': 'Petersham',
    '01367': 'Rowe',
    '01368': 'Royalston',
    '01370': 'Shelburne',
    '01373': 'Deerfield',
    '01375': 'Sunderland',
    '01376': 'Montague',
    '01378': 'Warwick',
    '01379': 'Wendell',
    '01380': 'Wendell',
    '01420': 'Fitchburg',
    '01430': 'Ashburnham',
    '01431': 'Ashby',
    '01432': 'Ayer',
    '01434': 'Ayer',
    '01436': 'Templeton',
    '01438': 'Templeton',
    '01440': 'Gardner',
    '01441': 'Westminster',
    '01450': 'Groton',
    '01451': 'Harvard',
    '01452': 'Hubbardston',
    '01453': 'Leominster',
    '01460': 'Littleton',
    '01462': 'Lunenburg',
    '01463': 'Pepperell',
    '01464': 'Shirley',
    '01467': 'Harvard',
    '01468': 'Templeton',
    '01469': 'Townsend',
    '01470': 'Groton',
    '01471': 'Groton',
    '01472': 'Groton',
    '01473': 'Westminster',
    '01474': 'Townsend',
    '01475': 'Winchendon',
    '01477': 'Winchendon',
    '01501': 'Auburn',
    '01503': 'Berlin',
    '01504': 'Blackstone',
    '01505': 'Boylston',
    '01506': 'Brookfield',
    '01507': 'Charlton',
    '01508': 'Charlton',
    '01509': 'Charlton',
    '01510': 'Clinton',
    '01515': 'East Brookfield',
    '01516': 'Douglas',
    '01517': 'Princeton',
    '01518': 'Sturbridge',
    '01519': 'Grafton',
    '01520': 'Holden',
    '01521': 'Holland',
    '01522': 'Holden',
    '01523': 'Lancaster',
    '01524': 'Leicester',
    '01525': 'Northbridge',
    '01526': 'Sutton',
    '01527': 'Millbury',
    '01529': 'Millville',
    '01531': 'New Braintree',
    '01532': 'Northborough',
    '01534': 'Northbridge',
    '01535': 'North Brookfield',
    '01536': 'Grafton',
    '01537': 'Oxford',
    '01538': 'Uxbridge',
    '01540': 'Oxford',
    '01541': 'Princeton',
    '01542': 'Leicester',
    '01543': 'Rutland',
    '01545': 'Shrewsbury',
    '01546': 'Shrewsbury',
    '01550': 'Southbridge',
    '01560': 'Grafton',
    '01561': 'Lancaster',
    '01562': 'Spencer',
    '01564': 'Sterling',
    '01566': 'Sturbridge',
    '01568': 'Upton',
    '01569': 'Uxbridge',
    '01570': 'Webster',
    '01571': 'Dudley',
    '01580': 'Westborough',
    '01581': 'Westborough',
    '01582': 'Westborough',
    '01583': 'West Boylston',
    '01585': 'West Brookfield',
    '01586': 'Millbury',
    '01588': 'Mansfield',
    '01590': 'Sutton',
    '01601': 'Worcester',
    '01602': 'Worcester',
    '01603': 'Worcester',
    '01604': 'Worcester',
    '01605': 'Worcester',
    '01606': 'Worcester',
    '01607': 'Worcester',
    '01608': 'Worcester',
    '01609': 'Worcester',
    '01610': 'Worcester',
    '01611': 'Leicester',
    '01612': 'Paxton',
    '01613': 'Worcester',
    '01614': 'Worcester',
    '01615': 'Worcester',
    '01653': 'Worcester',
    '01654': 'Worcester',
    '01655': 'Worcester',
    '01701': 'Framingham',
    '01702': 'Framingham',
    '01703': 'Framingham',
    '01704': 'Framingham',
    '01705': 'Framingham',
    '01718': 'Acton',
    '01719': 'Boxborough',
    '01720': 'Acton',
    '01721': 'Ashland',
    '01730': 'Bedford',
    '01731': 'Bedford',
    '01740': 'Bolton',
    '01741': 'Carlisle',
    '01742': 'Concord',
    '01745': 'Southborough',
    '01746': 'Holliston',
    '01747': 'Hopedale',
    '01748': 'Hopkinton',
    '01749': 'Hudson',
    '01752': 'Marlborough',
    '01754': 'Maynard',
    '01756': 'Mendon',
    '01757': 'Milford',
    '01760': 'Natick',
    '01770': 'Sherborn',
    '01772': 'Southborough',
    '01773': 'Lincoln',
    '01775': 'Stow',
    '01776': 'Sudbury',
    '01778': 'Wayland',
    '01784': 'Hopkinton',
    '01801': 'Woburn',
    '01803': 'Burlington',
    '01805': 'Burlington',
    '01806': 'Woburn',
    '01807': 'Woburn',
    '01808': 'Woburn',
    '01810': 'Andover',
    '01812': 'Andover',
    '01813': 'Woburn',
    '01815': 'Woburn',
    '01821': 'Billerica',
    '01822': 'Billerica',
    '01824': 'Chelmsford',
    '01826': 'Dracut',
    '01827': 'Dunstable',
    '01830': 'Haverhill',
    '01831': 'Haverhill',
    '01832': 'Haverhill',
    '01833': 'Georgetown',
    '01834': 'Groveland',
    '01835': 'Haverhill',
    '01840': 'Lawrence',
    '01841': 'Lawrence',
    '01842': 'Lawrence',
    '01843': 'Lawrence',
    '01844': 'Methuen',
    '01845': 'North Andover',
    '01850': 'Lowell',
    '01851': 'Lowell',
    '01852': 'Lowell',
    '01853': 'Lowell',
    '01854': 'Lowell',
    '01860': 'Merrimac',
    '01862': 'Billerica',
    '01863': 'Chelmsford',
    '01864': 'North Reading',
    '01865': 'Billerica',
    '01866': 'Billerica',
    '01867': 'Reading',
    '01876': 'Tewksbury',
    '01879': 'Tyngsborough',
    '01880': 'Wakefield',
    '01885': 'Boxford',
    '01886': 'Westford',
    '01887': 'Wilmington',
    '01888': 'Woburn',
    '01889': 'North Reading',
    '01890': 'Winchester',
    '01899': 'Andover',
    '01901': 'Lynn',
    '01902': 'Lynn',
    '01903': 'Lynn',
    '01904': 'Lynn',
    '01905': 'Lynn',
    '01906': 'Saugus',
    '01907': 'Swampscott',
    '01908': 'Nahant',
    '01910': 'Lynn',
    '01913': 'Amesbury',
    '01915': 'Beverly',
    '01921': 'Boxford',
    '01922': 'Newbury',
    '01923': 'Danvers',
    '01929': 'Essex',
    '01930': 'Gloucester',
    '01931': 'Gloucester',
    '01936': 'Hamilton',
    '01937': 'Hathorne',
    '01938': 'Ipswich',
    '01940': 'Lynnfield',
    '01944': 'Manchester-by-the-Sea',
    '01945': 'Marblehead',
    '01949': 'Middleton',
    '01950': 'Newburyport',
    '01951': 'Newbury',
    '01952': 'Salisbury',
    '01960': 'Peabody',
    '01961': 'Peabody',
    '01965': 'Beverly',
    '01966': 'Rockport',
    '01969': 'Rowley',
    '01970': 'Salem',
    '01971': 'Salem',
    '01982': 'Hamilton',
    '01983': 'Topsfield',
    '01984': 'Wenham',
    '01985': 'West Newbury',
    '02018': 'Norwell',
    '02019': 'Bellingham',
    '02020': 'Nantucket',
    '02021': 'Canton',
    '02025': 'Cohasset',
    '02026': 'Dedham',
    '02027': 'Dedham',
    '02030': 'Dover',
    '02031': 'Mansfield',
    '02032': 'Walpole',
    '02035': 'Foxborough',
    '02038': 'Franklin',
    '02040': 'Scituate',
    '02041': 'Duxbury',
    '02043': 'Hingham',
    '02044': 'Hingham',
    '02045': 'Hull',
    '02047': 'Scituate',
    '02048': 'Mansfield',
    '02050': 'Marshfield',
    '02051': 'Marshfield',
    '02052': 'Medfield',
    '02053': 'Medway',
    '02054': 'Millis',
    '02055': 'Scituate',
    '02056': 'Norfolk',
    '02059': 'Marshfield',
    '02060': 'Scituate',
    '02061': 'Norwell',
    '02062': 'Norwood',
    '02065': 'Marshfield',
    '02066': 'Scituate',
    '02067': 'Sharon',
    '02070': 'Wrentham',
    '02071': 'Walpole',
    '02072': 'Stoughton',
    '02081': 'Walpole',
    '02090': 'Westwood',
    '02093': 'Wrentham',
    '02108': 'Boston',
    '02109': 'Boston',
    '02110': 'Boston',
    '02111': 'Boston',
    '02112': 'Boston',
    '02113': 'Boston',
    '02114': 'Boston',
    '02115': 'Boston',
    '02116': 'Boston',
    '02117': 'Boston',
    '02118': 'Boston',
    '02119': 'Boston',
    '02120': 'Boston',
    '02121': 'Boston',
    '02122': 'Boston',
    '02123': 'Boston',
    '02124': 'Boston',
    '02125': 'Boston',
    '02126': 'Boston',
    '02127': 'Boston',
    '02128': 'Boston',
    '02129': 'Boston',
    '02130': 'Boston',
    '02131': 'Boston',
    '02132': 'Boston',
    '02133': 'Boston',
    '02134': 'Boston',
    '02135': 'Boston',
    '02136': 'Boston',
    '02137': 'Boston',
    '02138': 'Cambridge',
    '02139': 'Cambridge',
    '02140': 'Cambridge',
    '02141': 'Cambridge',
    '02142': 'Cambridge',
    '02143': 'Somerville',
    '02144': 'Somerville',
    '02145': 'Somerville',
    '02148': 'Malden',
    '02149': 'Everett',
    '02150': 'Chelsea',
    '02151': 'Revere',
    '02152': 'Winthrop',
    '02153': 'Medford',
    '02155': 'Medford',
    '02156': 'Medford',
    '02163': 'Boston',
    '02169': 'Quincy',
    '02170': 'Quincy',
    '02171': 'Quincy',
    '02176': 'Melrose',
    '02180': 'Stoneham',
    '02184': 'Braintree',
    '02185': 'Braintree',
    '02186': 'Milton',
    '02187': 'Milton',
    '02188': 'Weymouth',
    '02189': 'Weymouth',
    '02190': 'Weymouth',
    '02191': 'Weymouth',
    '02196': 'Boston',
    '02199': 'Boston',
    '02201': 'Boston',
    '02203': 'Boston',
    '02204': 'Boston',
    '02205': 'Boston',
    '02206': 'Boston',
    '02207': 'Boston',
    '02210': 'Boston',
    '02211': 'Boston',
    '02212': 'Boston',
    '02215': 'Boston',
    '02216': 'Boston',
    '02217': 'Boston',
    '02222': 'Boston',
    '02228': 'Boston',
    '02238': 'Cambridge',
    '02239': 'Cambridge',
    '02241': 'Boston',
    '02266': 'Boston',
    '02269': 'Quincy',
    '02283': 'Boston',
    '02284': 'Boston',
    '02293': 'Boston',
    '02295': 'Boston',
    '02297': 'Boston',
    '02298': 'Boston',
    '02301': 'Brockton',
    '02302': 'Brockton',
    '02303': 'Brockton',
    '02304': 'Brockton',
    '02305': 'Brockton',
    '02322': 'Avon',
    '02324': 'Bridgewater',
    '02325': 'Bridgewater',
    '02327': 'Pembroke',
    '02330': 'Carver',
    '02331': 'Duxbury',
    '02332': 'Duxbury',
    '02333': 'East Bridgewater',
    '02334': 'Easton',
    '02337': 'East Bridgewater',
    '02338': 'Halifax',
    '02339': 'Hanover',
    '02340': 'Hanover',
    '02341': 'Hanson',
    '02343': 'Holbrook',
    '02344': 'Middleborough',
    '02345': 'Plymouth',
    '02346': 'Middleborough',
    '02347': 'Lakeville',
    '02348': 'Lakeville',
    '02349': 'Middleborough',
    '02350': 'Hanson',
    '02351': 'Abington',
    '02355': 'Carver',
    '02356': 'Easton',
    '02357': 'Easton',
    '02358': 'Pembroke',
    '02359': 'Pembroke',
    '02360': 'Plymouth',
    '02361': 'Plymouth',
    '02362': 'Plymouth',
    '02364': 'Kingston',
    '02366': 'Carver',
    '02367': 'Plympton',
    '02368': 'Randolph',
    '02370': 'Rockland',
    '02375': 'Easton',
    '02379': 'West Bridgewater',
    '02381': 'Plymouth',
    '02382': 'Whitman',
    '02420': 'Lexington',
    '02421': 'Lexington',
    '02445': 'Brookline',
    '02446': 'Brookline',
    '02447': 'Brookline',
    '02451': 'Waltham',
    '02452': 'Waltham',
    '02453': 'Waltham',
    '02454': 'Waltham',
    '02455': 'Waltham',
    '02456': 'Littleton',
    '02457': 'Wellesley',
    '02458': 'Newton',
    '02459': 'Newton',
    '02460': 'Newton',
    '02461': 'Newton',
    '02462': 'Newton',
    '02464': 'Newton',
    '02465': 'Newton',
    '02466': 'Newton',
    '02467': 'Newton',
    '02468': 'Newton',
    '02471': 'Watertown',
    '02472': 'Watertown',
    '02474': 'Arlington',
    '02475': 'Arlington',
    '02476': 'Arlington',
    '02477': 'Watertown',
    '02478': 'Belmont',
    '02479': 'Belmont',
    '02481': 'Wellesley',
    '02482': 'Wellesley',
    '02492': 'Needham',
    '02493': 'Weston',
    '02494': 'Needham',
    '02495': 'Newton',
    '02532': 'Bourne',
    '02534': 'Bourne',
    '02536': 'Falmouth',
    '02537': 'Sandwich',
    '02538': 'Wareham',
    '02539': 'Edgartown',
    '02540': 'Falmouth',
    '02541': 'Falmouth',
    '02542': 'Bourne',
    '02543': 'Falmouth',
    '02552': 'Chilmark',
    '02553': 'Bourne',
    '02554': 'Nantucket',
    '02556': 'Falmouth',
    '02557': 'Oak Bluffs',
    '02558': 'Wareham',
    '02559': 'Bourne',
    '02561': 'Bourne',
    '02562': 'Bourne',
    '02563': 'Sandwich',
    '02564': 'Nantucket',
    '02565': 'Falmouth',
    '02568': 'Tisbury',
    '02571': 'Wareham',
    '02573': 'Tisbury',
    '02574': 'Falmouth',
    '02575': 'West Tisbury',
    '02576': 'Wareham',
    '02584': 'Nantucket',
    '02601': 'Barnstable',
    '02630': 'Barnstable',
    '02631': 'Brewster',
    '02632': 'Barnstable',
    '02633': 'Chatham',
    '02634': 'Barnstable',
    '02635': 'Barnstable',
    '02636': 'Barnstable',
    '02637': 'Barnstable',
    '02638': 'Dennis',
    '02639': 'Dennis',
    '02641': 'Dennis',
    '02642': 'Eastham',
    '02643': 'Orleans',
    '02644': 'Sandwich',
    '02645': 'Harwich',
    '02646': 'Harwich',
    '02647': 'Barnstable',
    '02648': 'Barnstable',
    '02649': 'Mashpee',
    '02650': 'Chatham',
    '02651': 'Eastham',
    '02652': 'Truro',
    '02653': 'Orleans',
    '02655': 'Barnstable',
    '02657': 'Provincetown',
    '02659': 'Chatham',
    '02660': 'Dennis',
    '02661': 'Harwich',
    '02662': 'Orleans',
    '02663': 'Wellfleet',
    '02664': 'Yarmouth',
    '02666': 'Truro',
    '02667': 'Wellfleet',
    '02668': 'Barnstable',
    '02669': 'Chatham',
    '02670': 'Dennis',
    '02671': 'Harwich',
    '02672': 'Barnstable',
    '02673': 'Yarmouth',
    '02675': 'Yarmouth',
    '02702': 'Freetown',
    '02703': 'Attleboro',
    '02712': 'Norton',
    '02713': 'Gosnold',
    '02714': 'Dartmouth',
    '02715': 'Dighton',
    '02717': 'Freetown',
    '02718': 'Taunton',
    '02719': 'Fairhaven',
    '02720': 'Fall River',
    '02721': 'Fall River',
    '02722': 'Fall River',
    '02723': 'Fall River',
    '02724': 'Fall River',
    '02725': 'Somerset',
    '02726': 'Somerset',
    '02738': 'Marion',
    '02739': 'Mattapoisett',
    '02740': 'New Bedford',
    '02741': 'New Bedford',
    '02742': 'New Bedford',
    '02743': 'Acushnet',
    '02744': 'New Bedford',
    '02745': 'New Bedford',
    '02746': 'New Bedford',
    '02747': 'Dartmouth',
    '02748': 'Dartmouth',
    '02760': 'North Attleboro',
    '02761': 'North Attleboro',
    '02762': 'Plainville',
    '02763': 'Attleboro',
    '02764': 'Dighton',
    '02766': 'Norton',
    '02767': 'Raynham',
    '02768': 'Raynham',
    '02769': 'Rehoboth',
    '02770': 'Rochester',
    '02771': 'Seekonk',
    '02777': 'Swansea',
    '02779': 'Berkley',
    '02780': 'Taunton',
    '02783': 'Taunton',
    '02790': 'Westport',
    '02791': 'Westport',
    '05501': 'Andover',
    '05544': 'Andover'
 }

# Mappings of city name to Municipality name ONLY for cities that contain zipcodes that map to multiple municipalities.
# Used to disambiguate multi-muni zipcodes when City field is present.
CITY_MUNI_MAP = {
    'amherst': 'Amherst',
    'easthampton': 'Easthampton',
    'westfield': 'Westfield',
    'becket': 'Becket',
    'dalton': 'Dalton',
    'lanesboro': 'Lanesborough',
    'north adams': 'North Adams',
    'south egremont': 'Egremont',
    'athol': 'Athol',
    'bernardston': 'Bernardston',
    'charlemont': 'Charlemont',
    'chilmark': 'Chilmark',
    'alford': 'Alford',
    'aquinnah': 'Aquinnah',
    'clarksburg': 'Clarksburg',
    'hancock': 'Hancock',
    'hawley': 'Hawley',
    'leyden': 'Leyden',
    'montgomery': 'Montgomery',
    'mount washington': 'Mount Washington',
    'new ashford': 'New Ashford',
    'pelham': 'Pelham',
    'phillipston': 'Phillipston',
    'washington': 'Washington',
    'westhampton': 'Westhampton'
 }

def clean(zip_series: pd.Series) -> pd.DataFrame:
    """Standardize zipcode formatting in a pandas series

    Pandas will likely load zipcodes from an excel file as an object series
    mixing string and numeric types.  This function will cast all entries to
    string types, strip whitespace, left pad to a minimum of 5 characters with zeros, then
    validate the entry contains a valid zipcode using a regular expression.

    Return: pandas.DataFrame
    - Pandas series of cleaned zipcodes: str
    - Pandas series of cleaned zip4, or empty string if missing: str
    - Pandas series of valid zipcode indicators: boolean
    """
    # Valid zipcodes are 3-5 numeric digits, followed by an optional dash and 4 more digits, or an optional ".0" if the data has been cast as floats.
    valid_zipcode_regex = r"^([0-9]{3,5})(?:[.]0)?(?:-([0-9]{4})|-)?$"
    #The extract function will match this pattern, and extract the zip5 group into column 0 and the zip4 group into column 1.  NaN if group is not present or pattern isn't matched.
    res = zip_series.astype(str).str.strip().str.extract(valid_zipcode_regex)

    cleaned_zipcode_series = res[0].str.zfill(5).fillna('')
    cleaned_zip4_series = res[1].fillna('')

    valid_zipcode_series = cleaned_zipcode_series.str.match(valid_zipcode_regex)

    #Replace invalid value rows with original inputs
    cleaned_zipcode_series.loc[~valid_zipcode_series] = zip_series[~valid_zipcode_series]
    return pd.DataFrame(data={"zip_cleaned": cleaned_zipcode_series, "zip4_cleaned": cleaned_zip4_series, "zip_valid": valid_zipcode_series})

def validate_zip_town_row(row: dict, town_field: str, zip_field: str) -> pd.Series:
    """
    Use zipcodes library to append town data from zipcodes and validate existing zip/town inputs
    """
    INVALID_RESULT = pd.Series({
        "town": "INVALID",
        "municipality": "INVALID",
        "zip_exists": False,
        "town_valid": False,
    })

    muni = ZIP_MUNI_MAP.get(row[zip_field])
    if not muni:
        #Try looking up town_field in City/Muni map
        standardized_raw_town_name = str(row[town_field]).lower().strip()
        muni = CITY_MUNI_MAP.get(standardized_raw_town_name)
        if not muni:
            return INVALID_RESULT

    # From here on, we were at least able to look up a muni.  But we still want to determine whether the raw town data is valid
    INVALID_RESULT = pd.Series({
            "municipality": muni,
            "town": string.capwords(str(row[town_field]).strip()),
            "zip_exists": False,
            "town_valid": False,
        })

    try:
        zip_results = zipcodes.matching(row[zip_field])
    except TypeError:
        return INVALID_RESULT
    except ValueError:
        return INVALID_RESULT

    if len(zip_results) > 1:
        raise ValueError(f"Multimatch zipcode {row[zip_field]} encountered!")
    elif len(zip_results) == 0:
        return INVALID_RESULT
    else: # len(zip_results)==1

        zip_results = zip_results[0]
        standardized_raw_town_name = str(row[town_field]).lower().strip()
        standardized_town_name = zip_results["city"].lower().strip()
        standardized_acceptable_towns = [x.lower().strip() for x in zip_results["acceptable_cities"]]
        standardized_acceptable_towns.append(standardized_town_name)

        #Strict exact matching for now
        town_valid = standardized_raw_town_name in standardized_acceptable_towns
        output = {
            "municipality": muni,
            "town": zip_results["city"],
            "zip_exists": True,
            "town_valid": town_valid
        }
    return pd.Series(output)

def validate_zip_town(df: pd.DataFrame, town_field: str, zip_field: str) -> pd.DataFrame:
    """
    Use zipcodes library to append town data from zipcodes and validate existing zip/town inputs

    Return: pandas.DataFrame
    - Pandas series of town name from zipcodes library: str
    - Pandas series indicating zipcode exists: boolean
    - Pandas series indicating raw data town is valid given zipcode: boolean
    """
    return df.merge(df.apply(
                        lambda row: validate_zip_town_row(row, town_field, zip_field), axis=1
                        ),
                    left_index=True, right_index=True
                )

def valid_town_row(row: dict, town_field: str) -> pd.Series:
    standardized_town = string.capwords(row[town_field].strip())
    zip_list = zipcodes.filter_by(city=standardized_town, state='MA')
    town_valid = (len(zip_list)>0)

    muni_list = [ZIP_MUNI_MAP.get(z['zip_code']) for z in zip_list]
    unique_munis = list(pd.Series([m for m in muni_list if m is not None]).unique())
    if len(unique_munis) == 1:
        muni = unique_munis[0]
    else:
        muni = None

    if not muni:
        #Try looking up town_field in City/Muni map
        standardized_raw_town_name = str(row[town_field]).lower().strip()
        muni = CITY_MUNI_MAP.get(standardized_raw_town_name)
        if not muni:
            muni = "INVALID"
            town_valid = False

    output = {
            "municipality": muni,
            "town": standardized_town,
            "zip_exists": False,
            "zip_cleaned": "",
            "zip4_cleaned": "",
            "zip_valid": False,
            "town_valid": town_valid
        }
    return pd.Series(output)

def validate_town(df: pd.DataFrame, town_field: str) -> pd.DataFrame:
    """
    Use zipcodes library to verify town name is valid and append cleaned fields

    Return: pandas.DataFrame
    - Pandas series of standardize (Capitalized) town name from zipcodes library: str
    - Blank columns for zip_cleaned, zip4_cleaned, and zip_valid=False to comply with clean data schema
    - Pandas series indicating raw data town is valid in the sense of existing in the zipcode lookup table.
    """
    return df.merge(df.apply(lambda row: valid_town_row(row, town_field), axis=1), left_index=True, right_index=True)


import wikipedia
from aiogram import Bot, Dispatcher, executor, types
from aiogram.dispatcher.filters import Text
from aiogram.types import ReplyKeyboardMarkup, KeyboardButton, InlineKeyboardMarkup, InlineKeyboardButton

import wikipedia

wikipedia.set_lang("uz")

ikb = InlineKeyboardMarkup(inline_keyboard=[
    [InlineKeyboardButton("❤", callback_data="like")],
    [InlineKeyboardButton("😜", callback_data="dislike")]
])

TOKEN = "5993718094:AAFuH_ciUd-v1c4TtA2hF5wZHFSjRFIgXJk"
bot = Bot(token=TOKEN)
dp = Dispatcher(bot=bot)

kb = ReplyKeyboardMarkup(resize_keyboard=True)

rkb = ReplyKeyboardMarkup(resize_keyboard=True)
rkb.add(KeyboardButton(text="/help"))
rkb.add(KeyboardButton(text="/start"))

#
# @dp.message_handler(commands=["start"])
# async def start_command(massage: types.Message):
#     await massage.reply(text="Assalomu alaykum")
#
#
# @dp.message_handler(commands=["help"])
# async def start_command(massage: types.Message):
#     await massage.reply(text="o harfini tepasiga ' qo'yish kerak")

# @dp.message_handler()
# async def echo_ansver(massage: types.Message):
#     await massage.answer(text=massage.text.upper())

# @dp.message_handler(Text(equals="commandsss"))
# async def get_commma(massge: types.Message):
#     text = """
#     /help = yordam
#     /start = qayta boshlash
#     """
#
#     await massge.answer(text=text)

# @dp.message_handler(Text(equals=["salom"]))
# async def start_command(massage: types.Message):
#     await massage.reply(text="o harfini tepasiga ' qo'yish kerak")

# @dp.message_handler(Text(equals="s"))
# async def START_commma(massge: types.Message):
#     await bot.send_photo(chat_id=massge.from_user.id,
#                          photo="https://yandex.ru/images/search?pos=5&from=tabbar&img_url=http%3A%2F%2Fmobimg.b-cdn.net%2Fv3%2Ffetch%2F1f%2F1f409cc95f7ee2b80b3c440a4df4c9e2.jpeg%3Fw%3D1470%26r%3D0.5625&text=mountain&rpt=simage&lr=10335",
#                          caption="bu rasm sizga yoqdimi",
#                          reply_markup=ikb)

# @dp.message_handler(Text(equals="s"))
# async def START_commma(massge: types.Message):
#     await bot.send_photo(chat_id=massge.from_user.id,
#                          photo="https://yandex.ru/images/search?pos=5&from=tabbar&img_url=http%3A%2F%2Fmobimg.b-cdn.net%2Fv3%2Ffetch%2F1f%2F1f409cc95f7ee2b80b3c440a4df4c9e2.jpeg%3Fw%3D1470%26r%3D0.5625&text=mountain&rpt=simage&lr=10335",
#                          caption="bu rasm sizga yoqdimi",
#                          reply_markup=rkb)
#

# # @dp.callback_query_handler(text="like")
# # async def calll(callback: types.CallbackQuery):
# #     await callback.answer("you like it")
# #
# # @dp.callback_query_handler(text="dislike")
# # async def calll(callback: types.CallbackQuery):
# #     await callback.answer("you dislake it")
#
#
# @dp.callback_query_handler()
# async def calll(callback: types.CallbackQuery):
#     if callback.data == 'like':
#         await callback.answer("you like it")
#     elif callback.data == "dislike":
#         await callback.answer("you dislike it")

@dp.message_handler()
async def sendw(massage: types.Message):
    try:
        qaytgan = wikipedia.summary(massage.text)
        await massage.answer(qaytgan)
    except:
        await massage.answer("bu mavzuga oid ma'lumot yo`q")

# @dp.message_handler(Text(equals=["manzil"]))
# async def slo(massage: types.Message):
#     await bot.send_location(chat_id=massage.from_user.id,
#                             latitude=41.287629, longitude=69.219397)

if __name__ == '__main__':
    executor.start_polling(dispatcher=dp, skip_updates=True)


import numpy as np
import matplotlib.pyplot as pl

def f(y):
    v = np.array([y[1], (-9.81/0.6) * np.sin(y[0])])
    return v

def rk():
    g = 9.81
    l = 0.6
    y = np.array([(3/4)*np.pi, 0])
    t = 0
    t_series = [t]
    y_series = [(3/4)*np.pi]
    while t <= 10:
        # compute theta'(t_(i+1))
        k_1 = y
        k_2 = y + 0.01*0.5*f(k_1)
        k_3 = y + 0.01*(-f(k_1)+2*f(k_2))

        y = y + 0.01 * (f(k_1)+4*f(k_2)+f(k_3))/6
        y_series.append(y[0])

        t += 0.01
        t_series.append(t)

    pl.plot(t_series, y_series)
    pl.show()
    print(y)

print(np.sin((3/4)*np.pi))
rk()


import numpy as np
from tkinter import *
import cv2
import faces_train as ft
import os
import keys
import smtplib
import pyttsx3
import imghdr
from email.message import EmailMessage

root = Tk()
root.geometry('500x600')
heading = Label(root, text="Welcome! to the super secure face recognition system",
                font=('montserrat', 12, "bold"), fg="black").pack()

def speak(my_entry):
    engine = pyttsx3.init()
    voices = engine.getProperty('voices')
    engine.setProperty('voice', voices[1].id)
    engine.say(my_entry)
    engine.runAndWait()

def send_mails():
    with open('test.jpg', 'rb') as f:
        file_data = f.read()
        file_type = imghdr.what(f.name)
        file_name = f.name
    with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp:
        smtp.login(keys.Gmail(), keys.gmail_password())
        msg = EmailMessage()
        msg['Subject'] = "Alert !!!"
        msg['From'] = keys.Gmail()
        msg['To'] = keys.Alert_Gmail()
        msg.set_content("Someone tried to use your system. If it isn't you then please make necessary actions. Image "
                        "of that person is attached with this mail.")
        msg.add_attachment(file_data, maintype='image', subtype=file_type, filename=file_name)
        smtp.send_message(msg)

def on_click():
    my_label = Label(root, text="please, Look into the camera")
    my_label.pack()
    face_cascade = cv2.CascadeClassifier('cascades/data/haarcascade_frontalface_alt2.xml')
    cap = cv2.VideoCapture(0)
    speak("please, Look into the camera for 5 seconds to open Visual Studio Code")
    if cap.isOpened():
        ret, frame = cap.read()
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(gray, scaleFactor=1.5, minNeighbors=5)
        if faces == ():
            mylabel = Label(root, text="Sorry could not find your face try again")
            mylabel.pack()
            speak("Sorry could not find your face try again")
        else:
            for (x, y, w, h) in faces:
                print(x, y, w, h)
                roi_gray = gray[y:y + h, x:x + w]
                roi_color = frame[y:y + h, x:x + w]
                img_item = "test.jpg"
                cv2.imwrite(img_item, roi_color)
                a = ft.classify_face("test.jpg")
                print(ft.classify_face("test.jpg"))
                if a == ['Unknown']:
                    try:
                        mylabel = Label(root, text="Sorry Sir !!! you are not authenticated to enter the system")
                        mylabel.pack()
                        speak("Sorry Sir !!! you are not authenticated to enter the system")
                        send_mails()
                    except Exception as e:
                        print(e)
                        mylabel = Label(root, text="Sorry Sir !!! you are not authenticated to enter the system")
                        mylabel.pack()
                        speak("Sorry Sir !!! you are not authenticated to enter the system")

                else:
                    mylabel = Label(root, text="Welcome Sir !!!")
                    mylabel.pack()
                    s = "Welcome Sir !!!" + " opening Visual Studio Code please wait."
                    speak(s)
                    path = keys.VS_PATH()
                    os.startfile(path)

    cap.release()

my_button = Button(root, text="click here to open VS Code", command=on_click)
my_button.pack()

root.mainloop()


# Module:   gui.TypePage.py | [ Language Python ]
# Author:   Gaps | sGaps | ArtGaps
# LICENSE:  GPLv3 (available in ./LICENSE.txt)
# -----------------------------------------------
"""
    Defines the Type page of the Smart Menu.

    [:] Defined in this module
    --------------------------
    TypePage :: class
        Allows user to choose between quick or custom mode.

    [*] Author
     |- Gaps : sGaps : ArtGaps
"""
from os                 import path
from .MenuPage          import AlternativePage , ColorIconButton , TextBesideIcon
from PyQt5.QtCore       import pyqtSlot , pyqtSignal , Qt
from PyQt5.QtWidgets    import QPushButton , QToolButton , QWidget , QVBoxLayout , QSizePolicy
from PyQt5.QtGui        import QFont , QIcon

class TypePage( AlternativePage ):
    CDIR         = path.dirname( path.abspath(__file__) )
    CUSTOM       = f"{CDIR}/images/custom.svg"
    QUICK        = f"{CDIR}/images/quick.svg"

    WCUSTOM      = f"{CDIR}/images/w_custom.svg"
    WQUICK       = f"{CDIR}/images/w_quick.svg"
    type_changed = pyqtSignal( bool )

    def __init__( self , backP = None , nextP = None , parent = None ):
        super().__init__( backP    = backP  ,
                          nextP    = nextP  ,
                          parent   = parent ,
                          subTitle = "Step 3: Which method would you like?" )
        # Toggle
        self.is_quick = True

        # Middle (Both Buttons)
        self.quick  = ColorIconButton( "Quick"   , True , TypePage.QUICK  , TypePage.WQUICK  , icon_size = (92,92) )
        self.custom = ColorIconButton( "Custom " , True , TypePage.CUSTOM , TypePage.WCUSTOM , icon_size = (92,92) )

        self.bottom = QWidget()
        self.bottom.setSizePolicy( QSizePolicy.Expanding , QSizePolicy.Expanding )
        self.sublyt = QVBoxLayout( self.bottom )
        self.sublyt.addWidget( self.custom )
        self.sublyt.addWidget( self.quick  )

        # Layout Setup:
        self.layout.addWidget( self.bottom )

        # Connections:
        self.quick.released.connect ( self.press_quick  )
        self.custom.released.connect( self.press_custom )
        self.press_quick()

    @pyqtSlot()
    def press_quick( self ):
        # Visual
        self.quick.setChecked ( True  )
        self.custom.setChecked( False )
        # Logical
        self.is_quick = True
        self.type_changed.emit( self.is_quick )

    @pyqtSlot()
    def press_custom( self ):
        # Visual
        self.custom.setChecked( True  )
        self.quick.setChecked ( False )
        # Logical
        self.is_quick = False
        self.type_changed.emit( self.is_quick )

    @pyqtSlot( bool )
    def serve_negated_alternative_request( self , to_alternative ):
        if (not to_alternative) != self.isAlt:
            self.useAlternative()

    def getData( self ):
        return { "is-quick" : self.is_quick }


import numpy as np
from scipy.special import lpmv
# from scipy.special import binom
# import scipy.constants as c
# import math
# from scipy.special import assoc_laguerre as al

def angular_wave_func(m,l,theta,phi):
    p1 = (2*l+1)/(4*c.pi)
    p2 = np.math.factorial(l - abs(m))/np.math.factorial(l+abs(m))
    p3 = (math.e)**(((-1)**0.5)*m*phi)
    alf = lpmv(m, l, np.cos(theta))
    print (alf)
    cal = alf*(((p1*p2)**0.5)*p3)
    if m >0:
        return round(cal, 5)
    elif m <=0:
        return round(cal, 5)
    else:
        pass

import math
import cmath
import numpy as np

def angular_wave_func(m,l,theta,phi):
    if l==0:
        output=(1/(4*math.pi))**0.5
    elif l==1:
        if m==1:
            output=np.sqrt(3/(8*math.pi))*np.sin(theta)*np.exp(-1j*(phi))
        elif m==0:
            output=output=(3/(4*math.pi))**0.5*math.cos(theta)
        elif m==-1:
            output=math.sqrt(3/(8*math.pi))*math.sin(theta)*cmath.exp(-phi*1j)
    elif l==2:
        if m==2:
            output=math.sqrt(15/(32*math.pi))*(math.sin(theta))**2*(cmath.exp(2j*phi))
        elif m==1:
            output=-(math.sqrt(15/(8*math.pi))*math.sin(theta)*math.cos(theta)*cmath.exp(phi*1j))
        elif m==0:
            output=output=(5/(16*math.pi))**0.5*(3*(math.cos(theta))**2-1)
        elif m==-2:
            output=math.sqrt(15/(32*math.pi))*(math.sin(theta))**2*(cmath.exp(-2j*phi))
        elif m==-1:
            output=math.sqrt(15/(8*math.pi))*math.sin(theta)*math.cos(theta)*cmath.exp(phi*-1j)
    elif l==3:
        if m==3:
            output=-(math.sqrt(35/(64*math.pi))*(math.sin(theta))**3*cmath.exp(3j*phi))
        elif m==2:
            output=math.sqrt(105/(32*math.pi))*math.cos(theta)*(math.sin(theta))**2*cmath.exp(2j*phi)
        elif m==1:
            output=-(math.sqrt(21/(64*math.pi))*math.sin(theta)*(5*(math.cos(theta))**2-1)*cmath.exp(1j*phi))
        elif m==0:
            output=math.sqrt(7/(16*math.pi))*(5*(math.cos(theta))**3-3*math.cos(theta))
        elif m==-1:
            output=(math.sqrt(21/(64*math.pi))*math.sin(theta)*(5*(math.cos(theta))**2-1)*cmath.exp(-1j*phi))
        elif m==-2:
            output=math.sqrt(105/(32*math.pi))*math.cos(theta)*(math.sin(theta))**2*cmath.exp(-2j*phi)
        elif m==-3:
            output=output=(math.sqrt(35/(64*math.pi))*(math.sin(theta))**3*cmath.exp(-3j*phi))
    return np.round(output,5)

# print('angular_wave_func (0,0,0,0)')
# ans= angular_wave_func (0,0,0,0)
# print(ans)
# print('angular_wave_func (0,1,c.pi ,0)')
# ans= angular_wave_func (0,1,c.pi ,0)
# print(ans)
# print('angular_wave_func (1,1,c.pi/2,c.pi)')
# ans= angular_wave_func (1,1,c.pi/2,c.pi)
# print(ans)
# print('angular_wave_func (0,2,c.pi ,0)')
# ans= angular_wave_func (0,2,c.pi,0)
# print(ans)

# def radial_wave_func(n,l,r):
#     a=c.physical_constants['Bohr radius'][0]
#     coef = 2/(n*a)
#     rcoef = (2*r)/(n*a)

#     p = 2*l+1
#     q=n+l
#     q_p = n-l-1

#     p1 = coef**3
#     p2 =  np.math.factorial(q_p)/(2*n*(np.math.factorial(n+l)))
#     p3 = (math.e**(-rcoef/2))*((rcoef)**l)
#     cal = ((p1*p2)**0.5)*p3*al(rcoef, q_p, p)
#     normal_cal = cal/(a**-1.5)
#     return round(normal_cal, 5)

# def calf(p, q_p, x):
#     if p == 0:
#         return 1
#     elif p == 3 and q_p == 0:
#         return 6
#     else:
#         return 64

# def calf(n, k, x):
#     sum = 0
#     for m in range(0,n+1):
#         sum+= (-1)**m * np.math.factorial(n+k) / np.math.factorial(n-m) / np.math.factorial(k+m) / np.math.factorial(m) * x**m
#     final = sum
#     return final

import scipy.constants as c
import numpy as np

def radial_wave_func(n,l,r):
    a=c.physical_constants['Bohr radius'][0]
    if n==1:
        output=(2/(a**3)**0.5)*np.exp(-r/a)
    elif n==2:
        if l==1:
            output=(1/(24**0.5))*(a**(-3/2))*(r/a)*np.exp(-r/(2*a))
        elif l==0:
            output=(1/(2**0.5))*(a**(-3/2))*(1-(r/(2*a)))*np.exp(-r/(2*a))
    elif n==3:
        if l==0:
            output=(2/(81*(3)**0.5))*(a**(-3/2))*(27-18*(r/a)+2*(r/a)**2)*np.exp(-r/(3*a))
        elif l==1:
            output=(8/(27*(6**0.5)))*(a**(-3/2))*(1-r/(6*a))*(r/a)*np.exp(-r/(3*a))
        elif l==2:
            output=(4/(81*(30)**0.5))*(a**(-3/2))*(r/a)**2*np.exp(-r/(3*a))
    elif n==4:
        if l==0:
            output=0.25*(a**(-3/2))*(1-(3/4)*(r/a)+(1/8)*(r/a)**2-(1/192)*(r/a)**3)*np.exp(-r/(4*a))
        elif l==1:
            output=((5)**0.5/(16*(3**0.5)))*(a**(-3/2))*(r/a)*(1-0.25*(r/a)+(1/80)*(r/a)**2)*np.exp(-r/(4*a))
        elif l==2:
            output=(1/(64*(5**0.5)))*(a**(-3/2))*(r/a)**2*(1-(1/12)*(r/a))*np.exp(-r/(4*a))
        elif l==3:
            output=(1/(768*(35**0.5)))*(a**(-3/2))*(r/a)**3*np.exp(-r/(4*a))
    output1=(output)/(a**(-1.5))
    return round(output1,5)

b = c.physical_constants['Bohr radius'][0]
print('radial_wave_func (1,0,a)')
ans= radial_wave_func (1,0,b)
print(ans)
print('radial_wave_func (2,1,a)')
ans= radial_wave_func (2,1,b)
print(ans)
print('radial_wave_func (2,1,2*a)')
ans= radial_wave_func (2,1,2*b)
print(ans)
print('radial_wave_func (3,1,2*a)')
ans= radial_wave_func (3,1,2*b)
print(ans)

# radial_wave_func (1,0,a) #p=1, q-p= 0
# 0.73576
# radial_wave_func (2,1,a) #p=3, q-p = 0
# 0.12381
# radial_wave_func (2,1,2*a) #p=3, q-p = 0
# 0.15019
# radial_wave_func (3,1,2*a) #p=3, q-p = 1
# 0.08281


#!/usr/bin/python3
"""a script 14-model_city_fetch_by_state.py that prints
all City objects from the database hbtn_0e_14_usa"""
import sys
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from model_state import State, Base
from model_city import City

if __name__ == "__main__":
    user, pwd, db = \
        sys.argv[1], sys.argv[2], sys.argv[3]
    url = 'mysql+mysqldb://{}:{}@localhost:3306/{}'
    url = url.format(user, pwd, db)
    engine = create_engine(url, pool_pre_ping=True)
    Session = sessionmaker(bind=engine)
    with Session() as session:
        query = session.query(State, City)\
            .filter(State.id == City.state_id)\
            .order_by(City.id)\
            .all()
        for state, city in query:
            print("{}: ({}) {}".format(state.name, city.id, city.name))

    engine.dispose()


"""
Models for "project parameters" package.
"""

from sqlalchemy import func, Index

from app import db
from app.base.models import BaseModel
# related model imports done in toolkit/__init__

class ProjectParameter(BaseModel):

    __tablename__ = 'project_parameter'

    created_by = db.Column(db.BigInteger, db.ForeignKey(
        'user.id', name='project_parameter_created_by_fkey'), nullable=False)
    updated_by = db.Column(db.BigInteger, db.ForeignKey(
        'user.id', name='project_parameter_updated_by_fkey'), nullable=False)

    # account id of the project
    account_id = db.Column(db.BigInteger, db.ForeignKey(
        'account.id', name='project_parameter_account_id_fkey'),
        nullable=False)
    project_id = db.Column(db.BigInteger, db.ForeignKey(
        'project.id', name='project_parameter_project_id_fkey',
        ondelete='CASCADE'), nullable=False)
    parent_parameter_name = db.Column(db.String(256), nullable=False)
    parameter_name = db.Column(db.String(256), nullable=False)
    parameter_value = db.Column(db.String(256))
    started_at = db.Column(db.DateTime)
    ended_at = db.Column(db.DateTime)
    level = db.Column(db.Integer, default=1)

    # multi column
    __table_args__ = (
        # unique lower case index, i.e case-insensitive unique constraint
        Index('ci_project_id_parent_parameter_name_parameter_name_key',
              project_id, func.lower(parent_parameter_name),
              func.lower(parameter_name), unique=True),
    )

    # relationships
    project = db.relationship('Project', backref=db.backref(
        'project_parameters', lazy='dynamic', passive_deletes=True))
    creator = db.relationship('User', backref=db.backref(
        'project_parameters', lazy='dynamic'),
        foreign_keys='ProjectParameter.created_by')
    account = db.relationship('Account', backref=db.backref(
        'project_parameters', lazy='dynamic'))

    def __init__(self, project_id=None, parent_parameter_name=None,
                 account_id=None, created_by=None,
                 updated_by=None, parameter_name=None,
                 *args, **kwargs):
        self.project_id = project_id
        self.parent_parameter_name = parent_parameter_name
        self.parameter_name = parameter_name
        self.account_id = account_id
        self.created_by = created_by
        self.updated_by = updated_by
        super(ProjectParameter, self).__init__(*args, **kwargs)

    def __repr__(self):
        return '<ProjectParameter %r>' % (self.row_id)


# Wikipedia scraper
from django.shortcuts import render

import bs4 as bs
import urllib.request
import re
import nltk
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
from summa import summarizer
from summa import keywords

def viewSumma(request):
  url_topull = request.GET.get('url', 'https://en.wikipedia.org/wiki/Machine_learning')

  scraped_data = urllib.request.urlopen(url_topull)
  article = scraped_data.read()

  parsed_article = bs.BeautifulSoup(article,'lxml')

  paragraphs = parsed_article.find_all('p')

  article_text = ""

  for p in paragraphs:
      article_text += p.text

  summary = summarizer.summarize(article_text,ratio=0.05)

  print("Data pull done")

  print("==================================SUMMARY===================================")
  print (summary)

  print("==================================KEYWORDS===================================")
  print (keywords.keywords(article_text,ratio=0.5))

  context = {
      'title': 'Summa',
      'summary': [summary],
  }
  return render(request, 'home.html', context)


# A rough test of formatting as Markdown
__all__ = ['MarkdownWriter']

from . import doc_tree, document, render, tree_formatter
from .format_html import HtmlWriter
import regex as re # type: ignore

from typing import *

class MarkdownFormatter(tree_formatter.TreeFormatter):
	def __init__(self):
		super().__init__()

	def text(self, text: str):
		#TODO: Escape
		self.write( text )

class StackItem:
	def __init__(self, node : doc_tree.Node ):
		self.node = node
		self.is_flow = False

"""
This is focused on dev.to flavoured Markdown at the moment, which seems pretty much similar to GitHub styled markdown. We'll have to add custom emitters for different markdown eventually (sharing a base class)
"""
class MarkdownWriter(render.Writer):

	def __init__(self):
		super()
		self.output = MarkdownFormatter()
		self.notes = []
		self.stack : List[StackItem] = []

	def render(self, doc : document.Document ) -> str:
		if doc.root is not None:
			doc.root.visit( self )

		self._write_notes()
		return self.output.value

	def render_node( self, node : doc_tree.Node ) -> str:
		node.visit( self )
		return self.output.value

	def _render_node_children( self, node : doc_tree.Node ) -> str:
		node.visit_children( self )
		return self.output.value

	def enter( self, node : doc_tree.Node ) -> bool:
		self.output.open_context()
		self.stack.append( StackItem(node) )
		return self._write_node( node )

	def exit( self, node : doc_tree.Node ) -> None:
		back = self.stack.pop()
		assert back.node == node
		self.output.end_context()

	def _write_node( self, node : doc_tree.Node ) -> bool:
		result = False
		def q( match_type, func : Callable[[Any], bool]) -> bool:
			nonlocal result
			if isinstance( node, match_type ):
				result = func( node )
				return True
			return False

		def fail():
			raise Exception( "unknown-node-type", node )

		_ = \
			q( doc_tree.Block, self._write_block ) or \
			q( doc_tree.Code, self._write_code ) or \
			q( doc_tree.Embed, self._write_embed ) or \
			q( doc_tree.Inline, self._write_inline ) or \
			q( doc_tree.Link, self._write_link ) or \
			q( doc_tree.ListItem, self._write_list_item ) or \
			q( doc_tree.List, self._write_list ) or \
			q( doc_tree.Note, self._write_note ) or \
			q( doc_tree.Paragraph, self._write_paragraph ) or \
			q( doc_tree.Section, self._write_section ) or \
			q( doc_tree.SectionTitle, self._write_section_title ) or \
			q( doc_tree.Text, self._write_text ) or \
			q( doc_tree.BlockMark, self._write_block_mark ) or \
			fail()

		return result

	inline_map = {
		"italic": ( "_", "_" ),
		"bold": ( "**", "**" ),
		"code": ( "`", "`" ),
		"none": ( "", "" ),
		"header": ( "**", ":**" ),
	}
	def _write_inline( self, node : doc_tree.Inline ) -> bool:
		if node.feature == doc_tree.feature_code:
			def format( text ) -> str:
				tick_len = 1 + _count_longest_backtick_chain( text )
				ticks = '`' * tick_len
				pre = ticks
				post = ticks
				if len(text) > 0 and text[0] == '`':
					pre += ' '
				if len(text) > 0 and text[-1] == '`':
					post = ' ' + post
				return f"{pre}{text}{post}"

			self.output.capture( format )
		else:
			fmt = type(self).inline_map.get(node.feature.name)
			if fmt is None:
				raise Exception( "unknown-inline-feature", node.feature.name )

			def format( text ) -> str:
				flow = _split_flow( text )
				#MYPY: failing to see the if statement from above
				assert fmt is not None
				return f"{flow[0]}{fmt[0]}{flow[1]}{fmt[1]}{flow[2]}"

			self.output.capture( format )

		return True

	def _write_paragraph( self, node : Union[doc_tree.Paragraph, doc_tree.ParagraphElement] ) -> bool:
		parent = self.stack[-2]
		if (isinstance(parent.node,doc_tree.NodeContainer) and parent.node.len_sub() != 1) or not parent.is_flow:
			self.output.section( '','\n\n' )
		# else we collapse the paragraph in a flow parent
		return True

	def _write_quote( self, node : doc_tree.Block ) -> bool:
		self.stack[-1].is_flow = True
		self.output.section( '>', '\n\n' )
		return True

	def _write_blurb( self, node : doc_tree.Block ) -> bool:
		self.output.write( '----\n\n' )
		self.stack[-1].is_flow = True
		self.output.section( '_', '_\n\n' )
		return True

	def _write_aside( self, node : doc_tree.Block ) -> bool:
		self.stack[-1].is_flow = True
		self.output.section( '>ðŸ’ ', '\n\n' )
		return True

	def _write_promote( self, node : doc_tree.Block ) -> bool:
		self.stack[-1].is_flow = True
		self.output.section( '>', '\n\n' )
		return True

	def _write_block( self, node : doc_tree.Block ) -> bool:
		if node.class_ == doc_tree.block_quote:
			return self._write_quote( node )
		elif node.class_ == doc_tree.block_blurb:
			return self._write_blurb( node )
		elif node.class_ == doc_tree.block_aside:
			return self._write_aside( node )
		elif node.class_ == doc_tree.block_promote:
			return self._write_promote( node )

		raise Exception( "unknown-block-type", node.class_.name )

	def _write_section( self, node : doc_tree.Section ) -> bool:
		return True

	def _write_section_title( self, node : doc_tree.SectionTitle ) -> bool:
		parent = self.stack[-2].node
		assert isinstance( parent, doc_tree.Section )
		self.output.section( f'{"#" * parent.level} ', '\n\n' )
		return True

	def _write_text( self, node : doc_tree.Text ) -> bool:
		self.output.text( node.text )
		return True

	def _write_link( self, node : doc_tree.Link ) -> bool:
		# TODO: more escaping
		self.output.section( '[', f']({node.url})' )
		return True

	def _write_note( self, node : doc_tree.Note ) -> bool:
		self.notes.append( node )
		number = len(self.notes)
		self.output.section( f'[{number}](#note-{number})', '' )
		return False

	def _write_notes( self ) -> None:
		if len(self.notes) == 0:
			return

		self.output.write( '\n----\n\n' )
		for index, note in enumerate(self.notes):
			self.output.write( '{}. '.format(index+1, index+1) )
			# TODO: This approach could be replaced with a separate visitor for formatting notes
			writer = MarkdownWriter()
			self.output.write( writer._render_node_children( note ) )
			self.output.write("\n")

	def _write_code( self, node : doc_tree.Code ) -> bool:
		# TODO: escape
		self.output.write( f"```{node.class_}\n{node.text}\n```\n\n" )
		return True

	def _write_list( self, node : doc_tree.List ) -> bool:
		# TODO: Fix this
		if not _is_simple_list( node ):
			writer = HtmlWriter()
			writer._write_node( node ) #TODO: private access
			self.output.section('', "\n" + writer.output.value + "\n")
			return True

		self.output.section( '', '\n' )
		return True

	def _write_list_item( self, node : doc_tree.ListItem ) -> bool:
		self.stack[-1].is_flow = True
		self.output.section( '- ', '\n' )
		return True

	def _write_embed( self, node : doc_tree.Embed ) -> bool:
		if node.class_ == doc_tree.EmbedClass.image:
			self.output.write( f"![]({node.url})\n\n" )
		elif node.class_ == doc_tree.EmbedClass.abstract:
			# Only for dev.to, this'll need a configuration
			self.output.write( f"{{% link {node.url} %}}\n\n" )
		else:
			raise Exception(f"Unsupported embed {node.class_.name}")

		return True

	def _write_block_mark( self, node : doc_tree.BlockMark ) -> bool:
		if node.class_ == doc_tree.MarkClass.minor_separator:
			self.output.write( '----' )
		else:
			assert False
		return True

def _count_longest_backtick_chain( text : str ) -> int:
	count = 0
	max_count = 0
	def update_max():
		nonlocal count, max_count
		max_count = max( count, max_count )
		count = 0

	for c in text:
		if c == '`':
			count += 1
		else:
			update_max()
	update_max()

	return max_count

def _is_simple_list( node : doc_tree.List ) -> bool:
	for sub in node.iter_sub():
		if not (sub.len_sub() == 1 and isinstance(sub.first_sub(), doc_tree.Paragraph)):
			return False
	return True

_split_flow_re = re.compile( r'(\s*)(.*)(\s*)' )
"""
	Splits text into (leading whitespace, stripped text, trailing whitespace)
	This is needed for Markdown features where spaces break feature parsing
"""
def _split_flow( text : str ) -> Tuple[str, str, str]:
	match = _split_flow_re.match( text )
	assert match != None
	return ( match.group(1), match.group(2), match.group(3) )


import torch
from torch import nn, optim
from torch.utils.data import DataLoader
from torchvision import transforms
from torchvision import datasets
from torch.utils.data import Dataset, DataLoader
from scipy.io import loadmat
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import torchvision
import torchvision.transforms as transforms

batch_size = 64
image_size = 28
val_path = "SVHN/testSet"

class RNN(nn.Module):
    def __init__(self, in_dim, hidden_dim, n_layer, n_class):
        super(RNN, self).__init__()
        self.n_layer = n_layer
        self.hidden_dim = hidden_dim
        self.lstm = nn.LSTM(in_dim, hidden_dim, n_layer, batch_first=True)
        self.classifier = nn.Linear(hidden_dim, n_class)

    def forward(self, x):
        out, _ = self.lstm(x)
        out = self.classifier(out[:, -1, :])
        return out

model_path = "RNN_NMNIST_MB_SVHN.pth"
Net = torch.load(model_path)

data_transforms = {
    "train": transforms.Compose(
        [
            transforms.Grayscale(num_output_channels=1),
            transforms.Resize((image_size, image_size)),
            # transforms.RandomAffine(15),
            # transforms.ColorJitter(
            #     brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1
            # ),
            # transforms.RandomRotation(15),
            transforms.ToTensor(),
        ]
    ),
    "val": transforms.Compose(
        [
            transforms.Grayscale(num_output_channels=1),
            transforms.Resize((image_size, image_size)),
            transforms.ToTensor(),
        ]
    ),
}

test_dataset = datasets.ImageFolder(
    val_path, transform=data_transforms["val"])
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)

import numpy as np
Net = torch.load(model_path)
test_acc = []
for step, (img, label) in enumerate(test_loader):
    b, c, h, w = img.size()
    img = img.squeeze(1)
    out = Net(img)
    _, pred = torch.max(out, 1)
    acc = (pred == label).sum().data.item() / batch_size
    test_acc.append(acc)
print("Test accuracy:", np.sum(test_acc) / len(test_acc))

## Read img file
image_size = 28
transformer = transforms.Compose([lambda x: Image.open(x).convert('L').resize((28, 28), Image.ANTIALIAS),
                                  transforms.ToTensor(),
                                  ])
img_path = "G-Capcha Img/Test_2/0.png"
img = 1 - transformer(img_path)
plt.imshow(Image.open(img_path).convert('L'))
img = img[0].unsqueeze(0)
out = Net(img)
_, pred = torch.max(out, 1)
print("Predict Class:", pred[0].numpy())


from flask import Flask, jsonify, request

app = Flask(__name__)

languagens = [{'name':'Python'}, {'name':'Java'}, {'name':'CSharp'}, {'name':'Ruby'}]

@app.route('/', methods=['GET'])
def init():
	return jsonify({'message' : 'When I was child'})

@app.route('/lang', methods=['GET'])
def returnAll():
	return jsonify({'Languagens' : languagens})

@app.route('/lang/<string:name>', methods=['GET'])
def returnOne(name):
	langs = [language for language in languagens if language['name']==name]
	return jsonify({'Language' : langs[0]})

@app.route('/lang', methods=['POST'])
def addOne():
	language = {'name' : request.json['name']}

	languagens.append(language)
	return jsonify({'language' : languagens})

@app.route('/lang/<string:name>', methods=['PUT'])
def editOne(name):
	langs = [language for language in languagens if language['name'] == name]

	langs[0]['name'] = request.json['name']
	return jsonify({'language' : langs[0]})

@app.route('/lang/<string:name>', methods=['DELETE'])
def removeOne(name):
	langs = [language for language in languagens if language['name'] == name]
	languagens.remove(langs[0])
	return jsonify({'languagens' : languagens})

if __name__ == '__main__':
	app.run(debug=True, port=8080)


import dgl
import torch

from ark_nlp.factory.task import SequenceClassificationTask

class TextLevelGCNTask(SequenceClassificationTask):
    """
    TextLevelGCN文本分类任务的Task

    Args:
        module: 深度学习模型
        optimizer: 训练模型使用的优化器名或者优化器对象
        loss_function: 训练模型使用的损失函数名或损失函数对象
        class_num (:obj:`int` or :obj:`None`, optional, defaults to None): 标签数目
        scheduler (:obj:`class`, optional, defaults to None): scheduler对象
        n_gpu (:obj:`int`, optional, defaults to 1): GPU数目
        device (:obj:`class`, optional, defaults to None): torch.device对象，当device为None时，会自动检测是否有GPU
        cuda_device (:obj:`int`, optional, defaults to 0): GPU编号，当device为None时，根据cuda_device设置device
        ema_decay (:obj:`int` or :obj:`None`, optional, defaults to None): EMA的加权系数
        **kwargs (optional): 其他可选参数
    """  # noqa: ignore flake8"

    def __init__(self, *args, **kwargs):

        super(TextLevelGCNTask, self).__init__(*args, **kwargs)

    def _train_collate_fn(
        self,
        batch
    ):
        batch_graph = []
        batch_label_ids = []

        for sample in batch:
            sample_graph = sample['sub_graph'].to(self.device)
            sample_graph.ndata['h'] = self.module.node_embed(torch.Tensor(sample['node_ids']).type(torch.long).to(self.device))
            sample_graph.edata['w'] = self.module.edge_embed(torch.Tensor(sample['edge_ids']).type(torch.long).to(self.device))

            batch_graph.append(sample_graph)
            batch_label_ids.append(sample['label_ids'])

        batch_graph = dgl.batch(batch_graph)

        return {'sub_graph': batch_graph, 'label_ids': torch.Tensor(batch_label_ids).type(torch.long)}

    def _evaluate_collate_fn(self, batch):
        return self._train_collate_fn(batch)


"""Uvmat interface (:mod:`fluidimage.run_from_xml`)
===================================================

From matlab, something like::

  system(['python -m fluidimage.run_from_xml ' path_to_instructions_xml])

.. todo::

   UVmat civserie: input and output.

"""

import argparse
import os
from time import time

import numpy as np
import scipy

from fluiddyn.util import import_class
from fluiddyn.util.paramcontainer import tidy_container
from fluidimage.topologies.piv import TopologyPIV

from . import (
    ParamContainer,
    SerieOfArraysFromFiles,
    config_logging,
    logger,
    reset_logger,
)

def tidy_uvmat_instructions(params):
    params._set_internal_attr("_tag", "instructions_uvmat")

    # get nicer names and a simpler organization...
    tidy_container(params)

    params.input_table = input_table = params.input_table.split(" & ")
    for i in range(1, len(input_table)):
        if input_table[i].startswith("/"):
            input_table[i] = input_table[i][1:]

    filename = "".join(input_table[2:])
    path_dir = os.path.join(*input_table[:2])
    path_file_input = os.path.abspath(os.path.join(path_dir, filename))
    path_dir_input = path_dir
    params._set_attrib("path_dir_input", path_dir_input)
    params._set_attrib("path_file_input", path_file_input)

    params._set_attrib("path_dir_output", path_dir_input + params.output_dir_ext)

    ir = params.index_range
    if not hasattr(ir, "incr_i"):
        ir._set_attrib("incr_i", 1)

    if not hasattr(ir, "first_i"):
        print("Warning: no attribute first_i in xml UVmat file.")
        ir._set_attrib("first_i", 1)

    if not hasattr(ir, "last_i"):
        print("Warning: no attribute last_i in xml UVmat file.")
        ir._set_attrib("last_i", None)

    if not hasattr(ir, "first_j"):
        strcouple = f"i:i+{ir.incr_i + 1}:{ir.incr_i}"
        ind_start = ir.first_i

        if ir.last_i is None:
            ind_stop = None
        else:
            ind_stop = ir.last_i - ir.incr_i + 1
    else:
        raise NotImplementedError

    params._set_attrib("strcouple", strcouple)
    params._set_attrib("ind_stop", ind_stop)
    params._set_attrib("ind_start", ind_start)

class ActionBase:
    def __init__(self, instructions, args):
        self.instructions = instructions

        # create the serie of arrays
        logger.info("Create the serie of arrays")
        self.serie_arrays = SerieOfArraysFromFiles(
            path=instructions.path_dir_input,
            index_slices=instructions.index_slices,
        )

class ActionAverage(ActionBase):
    """Compute the average and save as a png file."""

    def compute(self):
        instructions = self.instructions
        serie = self.serie_arrays
        # compute the average
        logger.info("Compute the average")
        a = serie.get_array_from_index(0)
        mean = np.zeros_like(a, dtype=np.float32)
        nb_fields = 0
        for a in serie.iter_arrays():
            mean += a
            nb_fields += 1
        mean /= nb_fields

        strindices_first_file = serie._compute_strindices_from_indices(
            *[indices[0] for indices in instructions.index_slices]
        )
        strindices_last_file = serie._compute_strindices_from_indices(
            *[indices[1] - 1 for indices in instructions.index_slices]
        )

        name_file = (
            serie.base_name
            + serie._separator_base_index
            + strindices_first_file
            + "-"
            + strindices_last_file
            + "."
            + serie.extension_file
        )

        path_save = os.path.join(instructions.path_dir_output, name_file)
        logger.info("Save in file:\n%s", path_save)
        scipy.misc.imsave(path_save, mean)
        return mean

def params_piv_from_uvmat_xml(instructions, args):
    params = TopologyPIV.create_default_params()

    params.series.path = instructions.path_file_input

    params.series.strcouple = instructions.strcouple
    params.series.ind_stop = instructions.ind_stop
    params.series.ind_start = instructions.ind_start

    params.saving.path = instructions.path_dir_output

    n0 = int(instructions.action_input.civ1.search_box_size.split("\t")[0])
    if n0 % 2 == 1:
        n0 -= 1

    params.piv0.shape_crop_im0 = n0

    if hasattr(instructions.action_input, "patch2"):
        params.multipass.subdom_size = (
            instructions.action_input.patch2.sub_domain_size
        )
    else:
        params.multipass.use_tps = False

    if hasattr(instructions.action_input, "civ2"):
        params.multipass.number = 2

    modif_fluidimage_params(params, args)

    return params

class ActionPIV(ActionBase):
    """Compute the average and save as a png file."""

    def __init__(self, instructions, args):
        self.instructions = instructions
        self.params = params_piv_from_uvmat_xml(instructions, args)
        logger.info("Initialize fluidimage computations with parameters:")
        logger.info(self.params._make_xml_text())
        self.topology = TopologyPIV(self.params)

    def compute(self):
        t = time()
        self.topology.compute(sequential=False)
        t = time() - t
        print(f"ellapsed time: {t}s")

actions_classes = {"aver_stat": ActionAverage, "civ_series": ActionPIV}

programs = ["uvmat", "fluidimage"]

def parse_args():
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument("path", help="Path file.", type=str, nargs="?")
    parser.add_argument(
        "-m",
        "--mode",
        help="‘ask’, ‘new_dir’, ‘complete’ or ‘recompute’.",
        type=str,
        default="ask",
    )

    return parser.parse_args()

def modif_fluidimage_params(params, args):
    try:
        params.saving.how = args.mode
    except AttributeError:
        pass

def main():

    reset_logger()
    config_logging("info")

    args = parse_args()

    path_instructions_xml = args.path
    logger.info(
        "\nFrom Python, start with instructions in xml file:\n%s",
        path_instructions_xml,
    )

    params = ParamContainer(path_file=path_instructions_xml)

    program = None

    try:
        params.Action.ActionName
    except AttributeError:
        pass
    else:
        program = "uvmat"

    try:
        program = params._value_text["program"]
    except (AttributeError, KeyError):
        pass

    if program not in programs:
        raise ValueError("Can not detect the program to launch.")

    print(args)

    print(f"using input file from {program}")

    if program == "uvmat":
        tidy_uvmat_instructions(params)
        action_name = params.action.action_name
        logger.info(
            'Check if the action "%s" is implemented by FluidImage', action_name
        )
        if action_name not in actions_classes.keys():
            raise NotImplementedError(
                'action "' + action_name + '" is not yet implemented.'
            )

        cls = actions_classes[action_name]
        action = cls(params, args)

    elif program == "fluidimage":
        cls = import_class(
            params._value_text["module"], params._value_text["class"]
        )
        modif_fluidimage_params(params, args)
        action = cls(params)

    return action.compute()

if __name__ == "__main__":

    result = main()


#!/usr/bin/env python
# coding=utf-8
from distutils.log import warn as printf
from json import dumps
from pprint import pprint

BOOKS={
    '0132269937':
    {
    'titile':'Core Python Programing',
    'edition':2,
    'year':2007
    },
    '0132356139':
    {
        'titile':'Python Web Development with Djano',
        'author':['Jeff Forcier','Paul Bissex',
                 'Wesley Chun']
    },
    '01371419':
    {
        'titile':'Python Fundamentals',
        'year':2009
    },
}

printf('*** Raw Dict')
print(BOOKS)

printf('***\n PERTTY_pERINTED DICT ***')
print(dumps(BOOKS,indent=4))


#!/usr/bin/python

import json
import urllib
import urllib2
import os.path

#authorization stuff
apikey=''
authorization=''
contenttype='application/json'
user=''
mytz='America/Los_Angeles'
jsondir = 'mapmyrun-json/'

headers = { 'Api-Key' : apikey,
            'Authorization' : authorization,
            'Content-Type' : contenttype }
limit=40
offset=0
activities='&activity_type=146&activity_type=208&activity_type=16'

base_url = 'https://oauth2-api.mapmyapi.com'

url = "/v7.0/workout/?user=" + user + "&offset=" + str(offset) + "&limit=" + str(limit) + activities
req = urllib2.Request('%s%s' % (base_url, url), None, headers)
r = urllib2.urlopen(req)
resp = json.loads(r.read())
r.close()

#get total number of items
total_count = resp.get('total_count')

while offset <= total_count:
	url = "/v7.0/workout/?user=" + user + "&offset=" + str(offset) + "&limit=" + str(limit) + activities
	req = urllib2.Request('%s%s' % (base_url, url), None, headers)
	r = urllib2.urlopen(req)
	resp = json.loads(r.read())
	r.close()
	workouts = resp.get('_embedded')
	for key,value in workouts.iteritems():
		for workout in value:
			links = workout.get('_links')
			self = links.get('self')
			id = self[0].get('id')
			jsonfile = jsondir + id + ".json"
			if not os.path.isfile(jsonfile):
				file = open(jsonfile,'w+')
				json.dump(workout,file)
				file.close()
	offset = offset + limit


from os import access
from flask import Flask, jsonify, Response
from werkzeug.wrappers import Request
from flask_jwt_extended import decode_token
from jwt.exceptions import ExpiredSignatureError
import src
app = src

class SimpleMiddleWare(object):
    """
        Simple WSGI middleware
    """
    def __init__(self, app):
        self.app = app
    def __call__(self, environ, start_response):
        request = Request(environ)
        try:
            if 'Authorization' in request.headers:
                token = request.headers.get('Authorization').split(' ')[1]
                with app.create_app().app_context():
                    payload = decode_token(token)
                environ['payload'] = payload
            return self.app(environ, start_response)
        except ExpiredSignatureError as e:
            res = Response("Token expired", mimetype="application/json", status=401)
            return res(environ, start_response)


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from sklearn.datasets import make_regression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import BaggingClassifier, RandomForestClassifier
from Librairie_version16062020 import*
import argparse
import functools
import random
import scipy.stats
from scipy import *

parser = argparse.ArgumentParser()
parser.add_argument('--fich', metavar='L', type=str, nargs='+',
                    help='le dataset')
parser.add_argument('--na', metavar='N', type=str, nargs='+',
                    help='gestion na')
parser.add_argument('--target', metavar='T', type=str, nargs='+',
                    help='target attribut')
parser.add_argument('--sep', metavar='L', type=str, nargs='+',
                    help='le séparateur')
parser.add_argument('--att', nargs='+', type=str)

args = parser.parse_args()

fich=tuple(args.fich)[0]

se=tuple(args.sep)[0]

#Lecture du fichier csv selon le séarateur
if (se=="space"):
	data = pd.read_csv('/home/ali/stage/public/clustering/datasets/'+fich,sep ='\s+')
elif (se=="tab"):
	data = pd.read_csv('/home/ali/stage/public/clustering/datasets/'+fich,sep ='\t')
elif (se=="comma"):
	data = pd.read_csv('/home/ali/stage/public/clustering/datasets/'+fich,sep =',')
elif (se=="semicolon"):
	data = pd.read_csv('/home/ali/stage/public/clustering/datasets/'+fich,sep =';')
else:
	data = pd.read_csv('/home/ali/stage/public/clustering/datasets/'+fich,sep =':')

fichier = open("tmp_cla2.txt","r")
fichier2 = open("att_values.txt","r")
numb_att = fichier.readline()

fichier2.readline()
fichier2.readline()

#Initiation de la liste des caractéristiques
col = []

#Initiation de la liste des valeurs des caractéristiques
val = []

for i in range (int(numb_att)):
    col.append(fichier.readline())
    val.append(fichier2.readline())

for i in range (int(numb_att)):
    col[i]=col[i][:-1]
    val[i]=val[i][:-1]

#Suppression de la valeur donnée au Target
for i in range (int(numb_att)):
    if (col[i]==tuple(args.target)[0][:-1]):
        val.pop(i)

for i in range (len(val)):
    val[i]=float(val[i])

data = data[col]

#Gestion des valeurs manquantes selon la méthode choisie par l'utilisateur
if(tuple(args.na)[0]=="remplacer"):
    for col in data.columns:
        data[col]=data[col].fillna(value=data[col].mean())
else:
    data = data.dropna(axis=0)

X = data

#Séparation du Target du reste des caractéristiques
y = data[tuple(args.target)[0][:-1]]

X = data.drop([tuple(args.target)[0][:-1]],axis=1)

#Passage aux matrices Numpy
Y2 = np.array(y)

Y2=Y2.reshape(Y2.shape[0], 1)

X2=np.array(X).reshape(Y2.shape[0],int(numb_att)-1)

M = np.hstack((np.ones(Y2.shape),X))

theta = np.random.randn(int(numb_att), 1)

t1= np.transpose(M).dot(M)
t2= np.transpose(Y2).dot(M)
t3= np.linalg.inv(t1)
theta_final=t3.dot(np.transpose(t2))

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def net_input(theta, x):
    return np.dot(x, theta)

def probability(theta, x):
    return sigmoid(net_input(theta, x))

def predict(x):
    return probability(theta_final, x)

def accuracy(x, actual_classes, probab_threshold=0.5):
    predicted_classes = (predict(x) >=
                         probab_threshold).astype(int)
    predicted_classes = predicted_classes.flatten()
    accuracy = np.mean(predicted_classes == actual_classes)
    return accuracy

x = np.array(val).reshape(1,int(numb_att)-1)
val = [1] + val
x2 = np.array(val).reshape(1,int(numb_att))
print(round(accuracy(M, Y2),5))
print([int(predict(x2)[0][0]>=0.5)])

#Les plus proches voisins
model = KNeighborsClassifier()
model.fit(X,y)
print(round(model.score(X,y),5))
print(model.predict(x))

#Bagging Classifier
model2 = BaggingClassifier(base_estimator=KNeighborsClassifier(),n_estimators=100)
model2.fit(X,y)
print(round(model2.score(X,y),5))
print(model2.predict(x))

#Random Forest
model3 = RandomForestClassifier(n_estimators=100)
model3.fit(X,y)
print(round(model3.score(X,y),5))
print(model3.predict(x))


#!/usr/bin/python3
#-utf-8
'''
Auteur: Olivier Bernard & Gabriel Desharnais
Date : 26 avril 2016
Entree: Clavier
Sortie: Moniteur
Programme: Programme pour ecrire des fichiers
           Programme pour lire des fichiers
           Programme pour tracer un graphique avec deux variables
'''

import numpy as np
from scipy.optimize import curve_fit
from matplotlib import pyplot as plt
import sys
import unicodedata
VERSION="0.0.0.0.0.0.4"
def formater(string):
    if (all(item in '0123456789.' for item in string)):
        string=string.replace('.',',')
    return string
def question(question_a_afficher,type_de_donnees,compteur=2,limites=-1,default=''):
    """Cette fonction permet d'éffectuer une requête d'entrée à l'instar de <<input>>
    sauf qu'elle retourne la réponse dans le type demandé en argument. Elle permet
    aussi de définir un nombre seuil, une fois ce seuil dépasser elle demmandera si
    l'utilisateur veut annuler la tâche en cours(entrer un nombre négatif pour
    désactiver cette fonction). Elle permet aussi d'imposer un nombre limites de
    tentatives après quoi elle annulera la tâche en cours (entrer un ombre négatif pour
    désactiver cette fonction). Elle permet aussi d'ajouter une valeur par défault si
    l'utilisateur entre une valeur vide<<''>>, alors c'est cette valeur qui sera considéré
    comme entrée."""
    #On devrait ajouter un truc qui permet de choisir comme type par défault string
    while True:
        compteur-=1
        limites-=1
        a=input(question_a_afficher)
        if a=='':
            a = default
        if type(0)==type_de_donnees:
            try:
                a=int(a)
                return a
            except ValueError:
                print("Vous devez entrer un entier.")
        elif type(0.0)==type_de_donnees:
            try:
                a=float(a)
                return a
            except ValueError:
                print("Vous devez entrer un nombre à virgule flotante.")
        elif type('sa')==type_de_donnees:
            try:
                a=str(a)
                return a
            except ValueError:
                print("Vous devez entrer une chaîne de charactères.")
        else:
            raise Exception("type de données inconnu")
        if compteur==0:
            compteur=1
            b=input('Voulez vous annuler le procéssus en cours? Entrez <<c>> pour annuler :')
            if b.upper()=="C":
                raise Exception("Annulation du procéssus par l'utilisateur")
        if limites == 0:
            #Ici un jour on devrait ajouter de quoi qui permet d'envoyer la valeur par défaut au lieu de juste annuler
            raise Exception("Annulation du procéssus (nombre limites de tentatives atteinte)")
def incertitude(*arg):
    tableauxpm[arg[0]][arg[1]]+=[arg[2],]
def importer_tab(*arg):
    if(len(arg)<3):
        arg=list(arg)+[",",]
    tableaux[arg[0]] = np.genfromtxt(arg[1], delimiter=arg[2], dtype=str)
    print("tableaux",arg[0],"importer")
def ploter_tableau(*_arg,header=0,x=0,y=1):
    for _a in _arg:
        if "=" in _a:
            exec(_a)

    plt.plot(tableaux[_arg[0]][int(header)+1:,int(x)].astype(float), tableaux[_arg[0]][int(header)+1:,int(y)].astype(float))
    plt.xlabel(tableaux[_arg[0]][int(header):,int(x)])
    plt.ylabel(tableaux[_arg[0]][int(header):,int(y)])

    #nom_graph = question("Entrer le nom du graphique :",type(''))
    #plt.savefig(nom_graph+'.png')               #Enregitre une image <<PNG>> du graphique
    plt.show()                                  #Affiche le graphique
def insert_droit(*arg):
    nom_tableau=arg[0]
    tableaux[nom_tableau]=np.concatenate((tableaux[nom_tableau][:,:int(arg[1])+1],np.empty((len(tableaux[nom_tableau][:,0]),1),dtype=object),tableaux[nom_tableau][:,int(arg[1])+1:]),axis=1)
    print("colonne insérée")
def insert_gauche(*arg):
    nom_tableau=arg[0]
    tableaux[nom_tableau]=np.concatenate((tableaux[nom_tableau][:,:int(arg[1])],np.empty((len(tableaux[nom_tableau][:,0]),1),dtype=object),tableaux[nom_tableau][:,int(arg[1]):]),axis=1)
    print("colonne insérée")
def afficher_variables():
    """Permet d'afficher toutes les tableaux en mémoire dans le projet courant"""
    for cle in tableaux.keys():
        print(cle)
def afficher_tab(*lvar):
    if not (lvar == ()):
        c=[]
        for l in (tableaux[lvar[0]])[0,:]:
            c+=[0]
        for r in tableaux[lvar[0]]:
            for l in range(len(r)):
                h=0
                if r[l]==None:
                    h=4
                else:
                    h=len(r[l])
                if h>c[l]:
                    c[l]=h
        for r in tableaux[lvar[0]]:
            print('-'*(sum(c)+len(c)+1))
            for l in range(len(r)):
                try:
                    h=len(r[l])
                except TypeError:
                   h=4
                print('|',formater(r[l]),end='',sep='')
                a=c[l]-h
                print(' '*a,end='')
            print('|')
        print('-'*(sum(c)+len(c)+1))
def supprimer_variable(*lvar):
    """Permet de supprimer un tableaux de la mémoire dans le projet courant"""
    if lvar== ():
        lvar=question("quelle variable voulez-vous supprimer? ",type('')).split()
    if lvar[0]=='ALL':
        global tableaux
        global nom_tableau
        tableaux={}
        nom_tableau={}
    else:
        for var in lvar:
            if var in tableaux:
                del tableaux[var]
            else:
                print('Variable«',var,'»inexistante')
def modifier_tab(*arg):
    if len(arg)<4:
        arg=list(arg)+["",]
    if len(arg)>3:
        tableaux[arg[0]][int(arg[1]),int(arg[2])]=arg[3]
def creer_tableau(*arg):
    """Cette fonction ajoute un tableau au dictionnaire tableaux."""

    #Le nom donnée par l'utilisateur servira à nommée le tableau à l'interne
    if len(arg)>2:
        nom_tableau = arg[0]
        lignes = int(arg[1])+1
        colonnes = int(arg[2])
    else:
        if len(arg)>0:
            nom_tableau = arg[0]
        else:
            nom_tableau = question("Entrer le nom que vous desirez donner au tableau : ",type(''))
        #L'utilisateur doit spécifié le nombre de colonnes et de lignes que doit contenir le tableau (plus tard ça vaudrait la peine de rendre ça facultatif)
        colonnes = question("Entrer le nombre de colonnes du tableau : ",type(0))
        lignes = question("Entrer le nombre de lignes du tableau (La ligne titre ne compte pas) :",type(0))+1

    #Création d'un tableau numpy pouvant contenir des chaines de charactères et étant du bon format
    tableaux[nom_tableau]=np.empty((lignes,colonnes),dtype=object)
    tableauxpm[nom_tableau]={}

    return nom_tableau
def importer_donnees_man(nom_tableau):
    #On devrait ajouter un truc qui vérifie si <<nom_tableau>> existe dans <<tableaux>>
    #if nom_tableau in tableaux:
    colonnes=tableaux[nom_tableau].shape[1]
    lignes=tableaux[nom_tableau].shape[0]
    print("\nVous devez donner un nom à chaqu'une des",colonnes,"colonnes\n")
    for n in range(colonnes):
        tableaux[nom_tableau][0,n]=question("Le nom de la colonne "+str(n)+"\n: ",type(''))

    #Entrees des données dans le tableau
    for l in range(1,lignes):
        for c in range(colonnes):
            tableaux[nom_tableau][l,c]=question("Entrez la valeur "+str(l)+" de la colonne "+tableaux[nom_tableau][0,c]+": ",type(''))

    return nom_tableau
def exporter_tableau(*arg):
    if len(arg)<1:
        print("Erreur. Veuillez fournir les arguments demmandés. Utillisez la fonction «AIDE» pour plus d'information" )
        return "_Erreur"
    if len(arg)<2:
        arg+=(arg[0],)
    if len(arg)<3:
        arg+=("txt",)
    if arg[0] in tableaux:                                                      #Vérifie si le tableau existe
        #écriture des données dans le fichiers
        #écriture des données sous format texte
        if arg[2].upper() in ("TXT","CSV"):
            if len(arg)>3:
                deli=arg[3]
            else:
                deli=','
            np.savetxt(arg[1],tableaux[arg[0]],fmt="%s",delimiter=deli)              #ecrit le tableau en chaines de charactères dans un fichier
            print ("Votre tableau est enregistre sous le nom «" + arg[1] + "»  dans le même répertoir qu'où le code se situe")
        #écriture des données sous format binaire
        elif arg[2].upper() in ("BIN",):
            np.save(arg[1],tableaux[arg[0]])              #ecrit le tableau en chaines de charactères dans un fichier
            print ("Votre tableau est enregistre sous le nom «" + arg[1] + "»  dans le même répertoir qu'où le code se situe")
        elif arg[2].upper() in ("LATEX","TEX"):
            a=export_tex(tableaux[arg[0]])
            b=open(arg[1],'w')
            b.write(a)
            b.close()
            print ("Votre tableau est enregistre sous le nom «" + arg[1] + "»  dans le même répertoir qu'où le code se situe")
        elif arg[2].upper() in ("MD",):
            a=export_md(tableaux[arg[0]])
            b=open(arg[1],'w')
            b.write(a)
            b.close()
            print ("Votre tableau est enregistre sous le nom «" + arg[1] + "»  dans le même répertoir qu'où le code se situe")
        elif arg[2].upper() in ("HTML","HTM"):
            a=export_html(tableaux[arg[0]])
            b=open(arg[1],'w')
            b.write(a)
            b.close()
            print ("Votre tableau est enregistre sous le nom «" + arg[1] + "»  dans le même répertoir qu'où le code se situe")
        else:
            print("Format d'exportation non supporté.")
        #On devrait ajouter d'autres type d'exportation
    else:
        print(arg[0],"n'existe pas. Impossible de l'exporter")

    return arg[0]
def entete(lignes,colonnes):
    print("Bienvenue dans l'assitant de création d'entête.")
    print("Votre tableau à",lignes,"lignes et",colonnes,"colones (Incluant la ligne de nom des colonnes)")
    ligne=[]
    while True:
        com=question(": ",type('')).split()
        if "FIN" ==com[0].upper():
            break
        elif "AJO_LIGNE"==com[0].upper():
            ligne.append((list(range(int(colonnes))),{},))
        elif "AFF"==com[0].upper():
            for lin in ligne:
                lig=''
                for c in set(lin[0]):
                    lig+='|'
                    lig+=' '*(lin[0].count(c)*3-2)
                    lig+='|'
                print(lig)
            print('| |'*int(colonnes))
        elif "FUS" ==com[0].upper():
            a=range(int(com[2]),int(com[3])+1)
            for f in a:
                ligne[int(com[1])][0][f]=int(com[2])
            ligne[int(com[1])][1][int(com[2])]=com[4]
        elif "NOM" ==com[0].upper():
            ligne[int(com[1])][1][ligne[int(com[1])][0][int(com[2])]]=com[3]
        else:
            print("commande inconnu")
    return ligne
def export_html(tab):
    fichier="
\n"
    for r in tab:
        fichier+="\t
\n"
        for e in r:
            fichier+="\t\t"+formater(e)+"	\n"
        fichier+="\t
\n"
    fichier+="
"
    return fichier
def export_md(tab):
    fichier="|"
    c=len(tab[0,:])
    for r in tab[0,:]:
        fichier+=r+"|"
    fichier+="\n|"
    for r in range(c):
        fichier+="---|"
    for l in tab[1:,:]:
        fichier+="\n|"
        for r in l:
            fichier+=formater(r)+"|"
    return fichier
def export_tex(tab):
    fichier="\\begin{tabular}{|"
    b=tab.shape[1] #Nombre de colonnes
    #Terminer l'entête
    for r in range(b):
        fichier+="c|"
    fichier+="}\n\\hline\n"
    #ajouter toutes les lignes
    for l in tab:
        #faire toutes les colonnes sauf la dernière
        for r in range(b-1):
            fichier+=formater(str(l[r]))+'&'
        fichier+=str(l[b-1])+"\\\\\n"
    fichier+="\\hline\n\\end{tabular}"
    #entete(lignes,colonnes)
    return fichier
def ecriture ():
    exporter_tableau(importer_donnees_man(creer_tableau()))
#Lecture d'une fichier
def lecture():
    print("Vous ne pouvez ouvrir que des fichier txt ou sans extensions avec ce programme")
    nom_doc = question("Entrez le nom complet (avec l'extension) du ficihier que vous voulez lire :",type(''))

    gd = open(nom_doc, 'r')
    print(gd.read())
    gd.close()

    graph = question("Voulez vous en faire une graphique (O: Oui, N: Non) :",type(''))

    if graph.upper() == 'O':
        graphique()
#Tracer de graphique
def graphique():
    nom_doc = question("Entrez le nom complet (avec l'extension) du fichier dont vous voulez faire un graphique :",type(''))

    gd = open(nom_doc, 'r')
    print(gd.read())
    gd.close()

    saut= question("Combien de ligne désirez vous sauter (combien de ligne ne sont pas des données) :",type(''))
    var =np.loadtxt(nom_doc,skiprows= int(saut) ,unpack=True)

    x = question("Entrez le numéros de la colonne qui sera la composante X du graphique :",type(0))

    y =question("Entrez le numéros de la colonne qui sera la composante Y du graphique :",type(0))

    varx = var[x-1]
    vary = var[y-1]
    axe_x =question("Entrer le titre de l'axe des x :",type(''))
    axe_y = question("Entrer le titre de l'axe des y :",type(''))

    plt.plot(varx, vary)
    plt.xlabel(axe_x)
    plt.ylabel(axe_y)

    nom_graph = question("Entrer le nom du graphique :",type(''))
    plt.savefig(nom_graph+'.png')               #Enregitre une image <<PNG>> du graphique
    plt.show()                                  #Affiche le graphique
def aide():
    print("Ceci est le menu d'aide\n")
    print("La version du logiciel: ",VERSION,"\n")
    print("Ce programme permet à l'utilisateur d'importer les données de différentes manières et d'en faire des tableaux ou des graphiques \n")
    print("afin de créer un tableau et de l'exporter, choisisez l'option 'E' dans le menu principal. Il vous faudra ensuite entrez les données manuellement")
    print("Pour Importer vos données depuis un fichier: choisisez l'option 'L' dans le menu principal")
    print("Pour afficher tous les tableaux en mémoire dans le projet courant «LIST_TABS»")
    print("Pour supprimer un tableau de la mémoire dans le projet courant «SUP_VAR»")
    print("Pour créer un tableau «NOUV_TAB»")
    print("Pour transcrire toutes vos donnée manuellement vers un tableau «REMP_TAB»")
    print("Pour exporter un tableau vers un fichier «EXP_TAB»")
    print("Pour importer un tableau depuis un fichier «IMP_TAB»")
    print("Pour afficher un graphique «PLOT_TAB»")
    print("Pour affciher un tableau «AFF_TAB»")
def quiter():
    exit()
def langue(*arg):
    global lalangue
    if (len(arg)<1):
        print("La langue est:",lalangue)
    else:
        if(arg[0].upper() in ["FR","EN"]):
            print("La langue était:",lalangue)
            lalangue=arg[0].upper()
            print("La langue est maintenant:", lalangue)
        else:
            print("«",arg[0].upper(),"»","n'est pas un format supporté")
#main, choix entre les différentes options
def main():
    #Créer le menu à l'aide d'un dictionnaire qui contient les fonctions
    menu={
        'E':ecriture,
        'L':lecture,
        'G':graphique,
        'A':aide,
        'Q':quiter,
        'LIST_TABS':afficher_variables,
        'AFF_TAB':afficher_tab,
        'SUP_VAR':supprimer_variable,
        'NOUV_TAB':creer_tableau,
        'REMP_TAB':importer_donnees_man,
        'MOD_TAB':modifier_tab,
        'IMP_TAB':importer_tab,
        'EXP_TAB':exporter_tableau,
        'PLOT_TAB':ploter_tableau,
        'INS_DROIT':insert_droit,
        'INS_GAU':insert_gauche,
        'INCERTITUDE':incertitude,
        'LANGUE':langue,
        'EE':entete
        }
    commande= input("Que voulez-vous faire (E: Ecrire un fichier, L: Lire un fichier, G: tracer de graphique, A: aide, Q: arret du programme): ")
    com_arg=commande.split()
    for x in range(1,len(com_arg)):
        if com_arg[x].upper().startswith("_ASK"):
            com_arg[x]=question(com_arg[x]+":",type(''))
    try:
        menu[com_arg[0].upper()](*com_arg[1:len(com_arg)])        #Ceci cherche dans le dictionnaire une fonction répertorier sous le nom de commande et essai de l'exécuter
    except KeyError:
        print("Vous n'avez pas entré une option valide.")
    except Exception as e:
        pass
        #e.args              # arguments stored in .args
    except TypeError as e:
        print("Vous avez donnez la mauvaise quantité d'argument à la fonction",e)
    main()
tableaux = {}
tableauxpm = {}
lalangue="FR"
if __name__ == '__main__':
  main()


# Author: Shiyang Jia

import os
import cv2
import numpy as np
import h5py

class Dataloader(object):
    """load data with ground truth for HairNet training and testing"""

    def __init__(self, data_path, batch_size):
        """
        Dataloader variables
        param			description				        size				type
        self.train_x 	list of training image 	   		(h, w, channel=2)	uint8 -> float64
        self.train_y    dict of ground truth        	(32, 32, x)         float64
        self.test_x     list of testing image 	   		(h, w, channel=2)	uint8 -> float64
        self.test_y		list of loss weight				(32, 32, x) 		float64
        """
        self.batch_size = batch_size
        self.train_x_dir = os.path.join(data_path, 'train_x_png')
        self.train_y_dir = os.path.join(data_path, 'train_y')
        self.test_x_dir = os.path.join(data_path, 'test_x_png')
        self.test_y_dir = os.path.join(data_path, 'test_y')
        self.pos_mean, self.pos_std, self.curv_mean, self.curv_std = get_mean_std(data_path)
        self.order = list(range(126))       # load test data first

    def load_image(self, image_path, begin, end):
        images = []
        file_list = os.listdir(image_path)
        file_list.sort()
        files = []
        for i in range(begin, end):
            files.append(file_list[self.order[i]])
        for file in files:
            img = cv2.imread(os.path.join(image_path, file), cv2.IMREAD_COLOR)
            img = img.astype(np.float64) / 255
            images.append(img[:, :, (2, 0)])  # (R, B)
        return images

    def load_gt(self, h5_dir, begin, end):
        gt = {'angle': [], 'pos': [], 'curv': [], 'weight': []}
        file_list = os.listdir(h5_dir)
        file_list.sort()
        files = []
        for i in range(begin, end):
            files.append(file_list[self.order[i]])
        for file in files:
            with h5py.File(os.path.join(h5_dir, file), 'r') as h5:
                gt['angle'].append(h5['angle'].value)
                gt['pos'].append(h5['pos'][:].reshape(32, 32, 300))
                gt['curv'].append(h5['curv'][:])
                gt['weight'].append(h5['weight'][:])
        return gt

    def flesh_batch_order(self):
        self.order = np.random.permutation(1800)    # set when training
        # self.order = list(range(1800))

    def get_train_batch(self, i):
        """used in training mode
        Args:
            i: the i-th batch according to self.order
        Return:
            encoder_input: [batch_size, 256, 256, 2]
            decoder_output: [batch_size, 32, 32, 500] -- weight + pos + curv
        """
        # load data
        n = self.batch_size
        begin, end = i * n, (i + 1)*n
        train_x = self.load_image(self.train_x_dir, begin, end)
        train_y = self.load_gt(self.train_y_dir, begin, end)

        # put all the data into big arrays
        encoder_input = np.zeros((n, 256, 256, 2), dtype=np.float64)
        decoder_output = np.zeros((n, 32, 32, 500), dtype=np.float64)    # weight + pos + curv
        for i in range(n):
            encoder_input[i] = train_x[i]
            norm_pos = (train_y['pos'][i] - self.pos_mean) / self.pos_std
            norm_curv = (train_y['curv'][i] - self.curv_mean) / self.curv_std
            decoder_output[i] = np.concatenate((train_y['weight'][i], norm_pos, norm_curv), axis=2)
        return encoder_input, decoder_output

    def get_test_data(self):
        """used in training mode as a test after each epoch"""

        test_x = self.load_image(self.test_x_dir, 0, 126)
        test_y = self.load_gt(self.test_y_dir, 0, 126)

        # put all the data into big arrays
        n = len(test_x)
        encoder_input = np.zeros((n, 256, 256, 2), dtype=np.float64)
        decoder_output = np.zeros((n, 32, 32, 500), dtype=np.float64)
        for i in range(n):
            encoder_input[i] = test_x[i]
            norm_pos = (test_y['pos'][i] - self.pos_mean) / self.pos_std
            norm_curv = (test_y['curv'][i] - self.curv_mean) / self.curv_std
            decoder_output[i] = np.concatenate((test_y['weight'][i], norm_pos, norm_curv), axis=2)

        return encoder_input, decoder_output, np.array(test_y['angle'])

def compute_mean_std(data_dir):
    """
    compute position mean/std and curvature mean/std among training samples
    store in mean_std.h5 file under data_dir and return
    """
    train_y_dir = os.path.join(data_dir, 'train_y')
    file_list = os.listdir(train_y_dir)

    # compute mean
    sum_pos = np.zeros((32, 32, 300), dtype=np.float64)
    sum_curv = np.zeros((32, 32, 100), dtype=np.float64)
    for file in file_list:
        with h5py.File(os.path.join(train_y_dir, file), 'r') as h5:
            sum_pos += h5['pos'][:].reshape(32, 32, 300)
            sum_curv += h5['curv'][:]

    pos_mean = sum_pos / 1800
    curv_mean = sum_curv / 1800

    # compute std
    sum_square_pos = np.zeros((32, 32, 300), dtype=np.float64)
    sum_square_curv = np.zeros((32, 32, 100), dtype=np.float64)

    for file in file_list:
        with h5py.File(os.path.join(train_y_dir, file), 'r') as h5:
            sum_square_pos += np.square(h5['pos'][:].reshape(32, 32, 300) - pos_mean)
            sum_square_curv += np.square(h5['curv'][:] - curv_mean)

    pos_std = np.sqrt(sum_square_pos / 1800)
    curv_std = np.sqrt(sum_square_curv / 1800)
    pos_std[pos_std == 0] = 1   # skip 0
    curv_std[curv_std == 0] = 1

    # write into h5 file under data_dir
    with h5py.File(os.path.join(data_dir, 'mean_std.h5'), 'w') as h5:
        h5.create_dataset('pos_mean', data=pos_mean)
        h5.create_dataset('pos_std', data=pos_std)
        h5.create_dataset('curv_mean', data=curv_mean)
        h5.create_dataset('curv_std', data=curv_std)

    return pos_mean, pos_std, curv_mean, curv_std

def get_mean_std(data_dir):
    """Return position & curvature mean, std (computed / loaded)"""

    file_path = os.path.join(data_dir, 'mean_std.h5')
    if os.path.exists(file_path):
        # read in mean/std
        with h5py.File(file_path, 'r') as h5:
            pos_mean = h5['pos_mean'][:]
            pos_std = h5['pos_std'][:]
            curv_mean = h5['curv_mean'][:]
            curv_std = h5['curv_std'][:]
        return pos_mean, pos_std, curv_mean, curv_std
    else:
        return compute_mean_std(data_dir)

def load_root(data_dir):
    """load hair root position and scalp-coordinate mask
    Args:
        data_dir: data_path/root_param.h5
    Return:
         root: (32, 32, 3) root position
         mask: [bool] (32, 32) true if there grows a strand
    """
    file_path = os.path.join(data_dir, 'root_param.h5')
    with h5py.File(file_path, 'r') as h5:
        root = h5['pos'][:]
        short_mask = h5['mask'][:]
        mask = short_mask > 0

    # put root in 32x32 array
    grid_root = np.zeros((32, 32, 3), dtype=float)
    idx = 0
    for i in range(32):
        for j in range(32):
            if mask[i, j]:
                grid_root[i, j] = root[idx]
                idx += 1

    return grid_root, mask

def load_real_image(data_dir):
    """ load test data
    Return:
        image_arr: (n, 256, 256, 2) numpy array
    """
    images = []
    files = os.listdir(data_dir)
    for file in files:
        img = cv2.imread(os.path.join(data_dir, file), cv2.IMREAD_COLOR)
        img = img.astype(np.float64) / 255
        images.append(img[:, :, (2, 0)])  # (R, B)

    # put all images into big array
    n = len(images)
    image_arr = np.zeros((n, 256, 256, 2), dtype=np.float64)
    for i in range(n):
        image_arr[i] = images[i]

    return image_arr


from langchain.agents import initialize_agent, ZeroShotAgent, AgentExecutor
from langchain import OpenAI, LLMChain, HuggingFacePipeline
from langchain.memory import ConversationBufferMemory
from langchain.chat_models import ChatOpenAI
from transformers import pipeline

from .utils import get_openai_api_key
from .tools import my_tools

def create_agent(
        model = 'text-davinci-003',
        tools = my_tools,
        temperature = 0,
        chat_history = False,
        task = 'text-generation',
        huggingface_pipeline_kwargs = None,
        huggingface_model_kwargs = None
):

    # Initiate regular model if not davinci
    if model in ['text-davinci-003']:
        llm = OpenAI(model_name = model, openai_api_key = get_openai_api_key(), temperature = temperature)

    # Initiate chat model if not davinci
    elif model in ['gpt-3.5-turbo', 'gpt-3.5-turbo-16k', 'gpt-4']:
        llm = ChatOpenAI(model_name = model, openai_api_key = get_openai_api_key(), temperature = temperature)

    # Else, try to instantiate the model using the pipeline interface
    else:
        llm = HuggingFacePipeline.from_model_id(
            model_id = model,
            task = task,
            model_kwargs = huggingface_model_kwargs,
            pipeline_kwargs = huggingface_pipeline_kwargs
        )

    if chat_history:
        prefix = """Have a conversation with a human, answering the following questions as best you can. You have access to the following tools:"""
        suffix = """Begin!"

        {chat_history}
        Question: {input}
        {agent_scratchpad}"""

        prompt = ZeroShotAgent.create_prompt(
            tools,
            prefix=prefix,
            suffix=suffix,
            input_variables=["input", "chat_history", "agent_scratchpad"],
        )
        memory = ConversationBufferMemory(memory_key="chat_history")
        llm_chain = LLMChain(llm=llm, prompt=prompt)
        agent = ZeroShotAgent(llm_chain=llm_chain, tools=tools, verbose=True)
        return AgentExecutor.from_agent_and_tools(
            agent=agent, tools=tools, verbose=True, memory=memory
        )
    else:
        return initialize_agent(
            tools = tools,
            llm = llm,
            verbose = True
        )