Archiving Data Files from Device Cloud

# Archive.py - download the XML files from the Device Cloud server
 
import os
import sys
import time
import urllib, urllib2
from urllib2 import URLError
 
# put your idigi account info here
IDIGI_URL = "http://developer.idigi.com"
USERNAME = "my_nama"
PASSWORD = "my_passa"
 
# your archive builds below this - if you use CRON to run this script,
# then you'll want the full path name
ARCHIVEBASE = '/home/idigi/archive'
 
# archive files are named with the modified-date from the Device Cloud storage
# in a form 'YYYYMMDD_HHMM' such as '20100812_1045'
 
# set to True to append the seconds, False to truncate them
ADD_SECS = False
 
# set True to force the seconds to be '00', ignored if ADD_SECS = False
ZERO_SECS = True
 
           # tuple of device id and storage directory tag name
DEV_LIST = [ ("00010000-00000000-03560210-13650987", 'x3_aio'),
             ("00000000-00000000-00409DFF-FF36EE2C", 'demo_solar'),
             ]
 
# In this example, the archive will store files as as:
#   /home/idigi/archive/x3_aio/x3_aio_20100812_1045.xml
#   /home/idigi/archive/x3_aio/x3_aio_20100812_1100.xml
#   /home/idigi/archive/demo_solar/demo_solar_20100812_1045.xml
#   /home/idigi/archive/demo_solar/demo_solar_20100812_1100.xml
# and so on
 
def log_into_idigi(username, password, idigiurl):
    '''Open the HTTP socket with login to idigi.com'''
    pwdManager = urllib2.HTTPPasswordMgrWithDefaultRealm()
    pwdManager.add_password(None, idigiurl, username, password)
    authHandler = urllib2.HTTPBasicAuthHandler(pwdManager)
    opener = urllib2.build_opener(authHandler)
    urllib2.install_opener(opener)
    return
 
def get_file_list(idigiurl, devid):
    '''Fetch a list of files in this device storage, which
    are returned as a list of tuples such as:
     [('data02.xml', '20100812_1045'),
      ('data03.xml', '20100812_1100')]
    These represent the file name in idigi and modified time
    '''
    dataurl = "%s/ws/data/~/%s/" % (idigiurl, devid)
 
    try:
        f = urllib2.urlopen(dataurl)
        data = f.read()
        # print data
        f.close()
 
    except URLError, e:
        print e.code
        print e.read()
        f.close()
        sys.exit(1)
 
    data = data.split('<')
    lst = []
    for ln in data:
        x = parse_fileinfo( ln)
        if x is not None:
            lst.append(x)
 
    return lst
 
def parse_fileinfo( ln):
    '''Return a tuple of file name and string of modified date,
    such as ('data02.xml', '20100812_1045')
    '''
 
    # you could expand this to use SAX or another XML parser, but
    # since we just extract 2 pieces of info per line, this brute
    # force method works and is likely faster
 
    if ln.startswith('exist:resource'):
        # print 'ln:%s' % ln
        x = ln.find('name=')
        if x >= 0:
            st = ln[x+6:].split('"')
            name = st[0]
            x = ln.find('modified=')
            if x >= 0:
                st = ln[x+10:].split('"')
                date = st[0]
                date = date[0:4] + date[5:7] + date[8:10] + '_' + \
                       date[11:13] + date[14:16]
                if ADD_SECS:
                    # then user wants the seconds in the name
                    if ZERO_SECS:
                        # then just treat as seconds
                        date += '00'
                    else: # else save actual
                        date += date[17:19]
                return (name, date)
 
    return None
 
def get_one_file(idigiurl, devid, filnam):
    '''Download one file as a string'''
    dataurl = "%s/ws/data/~/%s/%s" % (idigiurl, devid, filnam)
 
    try:
        f = urllib2.urlopen(dataurl)
        data = f.read()
        f.close()
    except URLError, e:
        print e.code
        print e.read()
        f.close()
        sys.exit(1)
 
    return data
 
def main():
 
    for dev_id, dev_nam in DEV_LIST:
        # verify the archive sub-directory exists
        arch = os.path.join(ARCHIVEBASE, dev_nam)
        if not os.path.exists(arch):
            print "Creating archive dir <%s>" % arch
            os.mkdir(arch)
        else:
            print "Device <%s> archive dir <%s> exists" % (dev_id,arch)
 
        # get the file list for this device        
        log_into_idigi(USERNAME, PASSWORD, IDIGI_URL)
        files = get_file_list(IDIGI_URL, dev_id)
        # print files                
 
        for fil_nam, fil_tim in files:
            save_file = os.path.join(arch, dev_nam + '_' + fil_tim + '.xml')
 
            if os.path.exists(save_file):
                print "Archive file <%s> already exists, skip download" % arch
 
            else:
                data = get_one_file(IDIGI_URL, dev_id, fil_nam)
                print data
                try:
                    print 'Saving file %s ...' % save_file,
                    f = open(save_file,'wb')
                    f.write(data)
                    f.close
                    print 'done.'
 
                except URLError, e:
                    print e.code
                    print e.read()
                    f.close()
 
if __name__ == '__main__':
    main()