#!/usr/bin/python import zipfile , sqlite3 , os , time from threading import Thread,Lock from thing import thing from thumb import thumb from stamp import stamp from download import download from unknown import unknown from image import image from like import like from tag import tag from folder import folder from author import author import random " storage folder " cache_base = 'cache' try: os.stat(cache_base) except: os.mkdir(cache_base) from thingi_page import thingi_index from thingi_page import thingi download_bits = "select t1.uuid,t2.ftype,t1.ftype from things as t1 , things as t2 where t1.parent = t2.uuid and t2.ftype = 'download';" " mapping for objects , ftype to python class" mapping = { 'thing' : thing, 'like' : like, 'jpg' : thumb, 'image' : image, 'download' : download, 'stamp' : stamp, 'thingi_index' : thingi_index, 'thingi' : thingi, 'tag' : tag, 'folder' : folder } " Sql thread " sql_lock = Lock() trans_counter = 0 class sql_base(Thread): def __init__(self,db_file): super(Thread, self).__init__() Thread.__init__(self) self.db = sqlite3.connect(db_file) self.c = self.db.cursor() def run(self): pass def query(self,query): #print(query) global trans_counter sql_lock.acquire() cu = self.c.execute(query) li = cu.fetchall() sql_lock.release() trans_counter = trans_counter + 1 if trans_counter == 10: self.commit() trans_counter = 0 return li def commit(self): self.db.commit() class missing(Exception): def __init__(self,value): self.value = value def __str__(self): return repr(self.value) "Base Storage class" class storage: def __init__(self,path='local',create=0): self.path = path try: os.stat(cache_base+'/'+path) except: if create: os.mkdir(cache_base+'/'+path) else: raise missing('no such folder') self.index_file = '' self.load_index() self.cache = {} self.users = {} self.tags = {} self.list_users() def load_index(self): self.index_file = cache_base + '/' + self.path + '/index.sdb' print(self.index_file) self.index = sql_base(self.index_file) self.index.query(thing('').get_table()) def extract(self): 'extract all the files into the cache' pass def load_all(self): "loads everything use sparingly" #l = self.index.query("select uuid from things where ftype='thing'") l = self.index.query("select uuid from things") for i in l: self.get_uuid(i[0]) def list_users(self): l = self.index.query('select distinct author from things group by author order by count(*) desc ') userlist = [] for i in l: userlist.append(i[0]) self.userlist = userlist def get_uuid(self,uuid): if self.cache.has_key(uuid): return self.cache[uuid] else: l = self.index.query("select * from things where uuid=='"+uuid+"'") if len(l) > 0: ftype = l[0][0] if mapping.has_key(ftype): t = mapping[ftype](self.index) else: #raise missing('no such type'+str(ftype)) t = unknown(uuid,self.index) t.load_data(l[0]) self.cache[uuid] = t return self.cache[uuid] else: raise missing('no uuid') def get_userid(self,userid): if self.users.has_key(userid): return self.users[userid] else: l = self.index.query("select uuid from things where author =='"+userid+"' order by id") if len(l) > 0: self.users[userid] = [] for i in l: self.users[userid].append(i[0]) return self.users[userid] else: raise missing('no such author') def get_thumb(self,the_thing): if the_thing.thumbnail == None: uu = the_thing.uuid l = self.index.query("select uuid from things where parent = '"+uu+"' and ftype = 'jpg' limit 1") for i in l: the_thing.thumbnail = self.get_uuid(i[0]) return True def load_thing(self,the_thing): if the_thing.loaded: return True else: uu = the_thing.uuid l = self.index.query("select uuid,ftype from things where parent = '"+uu+"'") d = {} for i in l: cthing = self.get_uuid(i[0]) if d.has_key(i[1]): d[i[1]].append(cthing) else: d[i[1]] = [cthing] the_thing.children = d the_thing.loaded = 1 return True def mark_broken(self,uuid): l = self.index.query("update things set processed = -1 where uuid = '"+uuid+"'") def mark_fetched(self,uuid): l = self.index.query("update things set processed = 1 where uuid = '"+uuid+"'") def mark_processed(self,uuid): l = self.index.query("update things set processed = 2 where uuid = '"+uuid+"'") def mark_seen(self,uuid,status=1): l = self.index.query("update things set status = "+str(int(status))+" where uuid = '"+uuid+"'") def process_list(self,level): l = self.index.query("select uuid from things where processed = "+str(level)+" order by timestamp desc limit 20") return l def stats(self): l = self.index.query("select count(*) from things where processed < 2 and processed >= 0") return l[0][0] def find(self,query): l = self.index.query("select uuid from things where processed > 0 and "+query) e = [] for i in l: e.append(self.get_uuid(i[0])) return e def chunk(self): l = self.index.query("select uuid from things where ftype='thingi' and status >= 0 order by status asc,timestamp desc limit 20") e = [] for i in l: e.append(i[0]) self.get_uuid(i[0]) return e