Package intermine :: Package lists :: Module listmanager
[hide private]
[frames] | no frames]

Source Code for Module intermine.lists.listmanager

  1  import weakref 
  2   
  3  # Use core json for 2.6+, simplejson for <=2.5 
  4  try: 
  5      import json 
  6  except ImportError: 
  7      import simplejson as json 
  8   
  9  import urllib 
 10  import codecs 
 11   
 12  from intermine.errors import WebserviceError 
 13  from intermine.lists.list import List 
14 15 -class ListManager(object):
16 """ 17 A Class for Managing List Content and Operations 18 ================================================ 19 20 This class provides methods to manage list contents and operations. 21 22 This class may be called itself, but all the useful methods it has 23 are also available on the Service object, which delegates to this class, while 24 other methods are more coneniently accessed through the list objects themselves. 25 26 NB: The methods for creating lists can conflict in threaded applications, if 27 two threads are each allocated the same unused list name. You are 28 strongly advised to use locks to synchronise any list creation requests (create_list, 29 or intersect, union, subtract, diff) unless you are choosing your own names each time 30 and are confident that these will not conflict. 31 """ 32 33 DEFAULT_LIST_NAME = "my_list_" 34 DEFAULT_DESCRIPTION = "List created with Python client library" 35 36 INTERSECTION_PATH = '/lists/intersect/json' 37 UNION_PATH = '/lists/union/json' 38 DIFFERENCE_PATH = '/lists/diff/json' 39 SUBTRACTION_PATH = '/lists/subtract/json' 40
41 - def __init__(self, service):
42 self.service = weakref.proxy(service) 43 self.lists = None 44 self._temp_lists = set()
45
46 - def refresh_lists(self):
47 """Update the list information with the latest details from the server""" 48 self.lists = {} 49 url = self.service.root + self.service.LIST_PATH 50 sock = self.service.opener.open(url) 51 data = sock.read() 52 sock.close() 53 list_info = json.loads(data) 54 if not list_info.get("wasSuccessful"): 55 raise ListServiceError(list_info.get("error")) 56 for l in list_info["lists"]: 57 l = ListManager.safe_dict(l) # Workaround for python 2.6 unicode key issues 58 self.lists[l["name"]] = List(service=self.service, manager=self, **l)
59 60 @staticmethod
61 - def safe_dict(d):
62 """Recursively clone json structure with UTF-8 dictionary keys""" 63 if isinstance(d, dict): 64 return dict([(k.encode('utf-8'), v) for k,v in d.iteritems()]) 65 else: 66 return d
67
68 - def get_list(self, name):
69 """Return a list from the service by name, if it exists""" 70 if self.lists is None: 71 self.refresh_lists() 72 return self.lists.get(name)
73
74 - def l(self, name):
75 """Alias for get_list""" 76 return self.get_list(name)
77
78 - def get_all_lists(self):
79 """Get all the lists on a webservice""" 80 if self.lists is None: 81 self.refresh_lists() 82 return self.lists.values()
83
84 - def get_all_list_names(self):
85 """Get all the names of the lists in a particular webservice""" 86 if self.lists is None: 87 self.refresh_lists() 88 return self.lists.keys()
89
90 - def get_list_count(self):
91 """ 92 Return the number of lists accessible at the given webservice. 93 This number will vary depending on who you are authenticated as. 94 """ 95 return len(self.get_all_list_names())
96
97 - def get_unused_list_name(self):
98 """ 99 Get an unused list name 100 ======================= 101 102 This method returns a new name that does not conflict 103 with any currently existing list name. 104 105 The list name is only guaranteed to be unused at the time 106 of allocation. 107 """ 108 list_names = self.get_all_list_names() 109 counter = 1 110 name = self.DEFAULT_LIST_NAME + str(counter) 111 while name in list_names: 112 counter += 1 113 name = self.DEFAULT_LIST_NAME + str(counter) 114 self._temp_lists.add(name) 115 return name
116
117 - def _get_listable_query(self, queryable):
118 q = queryable.to_query() 119 if not q.views: 120 q.add_view(q.root.name + ".id") 121 else: 122 # Check to see if the class of the selected items is unambiguous 123 up_to_attrs = set((v[0:v.rindex(".")] for v in q.views)) 124 if len(up_to_attrs) == 1: 125 q.select(up_to_attrs.pop() + ".id") 126 return q
127
128 - def _create_list_from_queryable(self, queryable, name, description, tags):
129 q = self._get_listable_query(queryable) 130 uri = q.get_list_upload_uri() 131 params = q.to_query_params() 132 params["listName"] = name 133 params["description"] = description 134 params["tags"] = ";".join(tags) 135 form = urllib.urlencode(params) 136 resp = self.service.opener.open(uri, form) 137 data = resp.read() 138 resp.close() 139 return self.parse_list_upload_response(data)
140
141 - def create_list(self, content, list_type="", name=None, description=None, tags=[], add=[]):
142 """ 143 Create a new list in the webservice 144 =================================== 145 146 If no name is given, the list will be considered to be a temporary 147 list, and will be automatically deleted when the program ends. To prevent 148 this happening, give the list a name, either on creation, or by renaming it. 149 150 This method is not thread safe for anonymous lists - it will need synchronisation 151 with locks if you intend to create lists with multiple threads in parallel. 152 153 @param content: The source of the identifiers for this list. This can be: 154 * A string with white-space separated terms. 155 * The name of a file that contains the terms. 156 * A file-handle like thing (something with a 'read' method) 157 * An iterable of identifiers 158 * A query with a single column. 159 * Another list. 160 @param list_type: The type of objects to include in the list. This parameter is not 161 required if the content parameter implicitly includes the type 162 (as queries and lists do). 163 @param name: The name for the new list. If none is provided one will be generated, and the 164 list will be deleted when the list manager exits context. 165 @param description: A description for the list (free text, default = None) 166 @param tags: A set of strings to use as tags (default = []) 167 @param add: The issues groups that can be treated as matches. This should be a 168 collection of strings naming issue groups that would otherwise be ignored, but 169 in this case will be added to the list. The available groups are: 170 * DUPLICATE - More than one match was found. 171 * WILDCARD - A wildcard match was made. 172 * TYPE_CONVERTED - A match was found, but in another type (eg. found a protein 173 and we could convert it to a gene). 174 * OTHER - other issue types 175 * :all - All issues should be considered acceptable. 176 This only makes sense with text uploads - it is not required (or used) when 177 the content is a list or a query. 178 179 @rtype: intermine.lists.List 180 """ 181 if description is None: 182 description = self.DEFAULT_DESCRIPTION 183 184 if name is None: 185 name = self.get_unused_list_name() 186 187 try: 188 ids = content.read() # File like thing 189 except AttributeError: 190 try: 191 with open(content) as c: # File name 192 ids = c.read() 193 except (TypeError, IOError): 194 try: 195 ids = content.strip() # Stringy thing 196 except AttributeError: 197 try: # Queryable 198 return self._create_list_from_queryable(content, name, description, tags) 199 except AttributeError: 200 try: # Array of idents 201 ids = "\n".join(map(lambda x: '"' + x + '"', iter(content))) 202 except AttributeError: 203 raise TypeError("Cannot create list from " + repr(content)) 204 205 uri = self.service.root + self.service.LIST_CREATION_PATH 206 query_form = { 207 'name': name, 208 'type': list_type, 209 'description': description, 210 'tags': ";".join(tags) 211 } 212 if len(add): query_form['add'] = [x.lower() for x in add if x] 213 214 uri += "?" + urllib.urlencode(query_form, doseq = True) 215 data = self.service.opener.post_plain_text(uri, ids) 216 return self.parse_list_upload_response(data)
217
218 - def parse_list_upload_response(self, response):
219 """ 220 Intepret the response from the webserver to a list request, and return the List it describes 221 """ 222 try: 223 response_data = json.loads(response) 224 except ValueError: 225 raise ListServiceError("Error parsing response: " + response) 226 if not response_data.get("wasSuccessful"): 227 raise ListServiceError(response_data.get("error")) 228 self.refresh_lists() 229 new_list = self.get_list(response_data["listName"]) 230 failed_matches = response_data.get("unmatchedIdentifiers") 231 new_list._add_failed_matches(failed_matches) 232 return new_list
233
234 - def delete_lists(self, lists):
235 """Delete the given lists from the webserver""" 236 all_names = self.get_all_list_names() 237 for l in lists: 238 if isinstance(l, List): 239 name = l.name 240 else: 241 name = str(l) 242 if name not in all_names: 243 continue 244 uri = self.service.root + self.service.LIST_PATH 245 query_form = {'name': name} 246 uri += "?" + urllib.urlencode(query_form) 247 response = self.service.opener.delete(uri) 248 response_data = json.loads(response) 249 if not response_data.get("wasSuccessful"): 250 raise ListServiceError(response_data.get("error")) 251 self.refresh_lists()
252
253 - def remove_tags(self, to_remove_from, tags):
254 """ 255 Add the tags to the given list 256 ============================== 257 258 Returns the current tags of this list. 259 """ 260 uri = self.service.root + self.service.LIST_TAG_PATH 261 form = {"name": to_remove_from.name, "tags": ";".join(tags)} 262 uri += "?" + urllib.urlencode(form) 263 body = self.service.opener.delete(uri) 264 return self._body_to_json(body)["tags"]
265
266 - def add_tags(self, to_tag, tags):
267 """ 268 Add the tags to the given list 269 ============================== 270 271 Returns the current tags of this list. 272 """ 273 uri = self.service.root + self.service.LIST_TAG_PATH 274 form = {"name": to_tag.name, "tags": ";".join(tags)} 275 resp = self.service.opener.open(uri, urllib.urlencode(form)) 276 body = resp.read() 277 resp.close() 278 return self._body_to_json(body)["tags"]
279
280 - def get_tags(self, im_list):
281 """ 282 Get the up-to-date set of tags for a given list 283 =============================================== 284 285 Returns the current tags of this list. 286 """ 287 uri = self.service.root + self.service.LIST_TAG_PATH 288 form = {"name": im_list.name} 289 uri += "?" + urllib.urlencode(form) 290 resp = self.service.opener.open(uri) 291 body = resp.read() 292 resp.close() 293 return self._body_to_json(body)["tags"]
294
295 - def _body_to_json(self, body):
296 try: 297 data = json.loads(body) 298 except ValueError: 299 raise ListServiceError("Error parsing response: " + body) 300 if not data.get("wasSuccessful"): 301 raise ListServiceError(data.get("error")) 302 return data
303
304 - def __enter__(self):
305 return self
306
307 - def __exit__(self, exc_type, exc_val, traceback):
309
310 - def delete_temporary_lists(self):
311 """Delete all the lists considered temporary (those created without names)""" 312 self.delete_lists(self._temp_lists) 313 self._temp_lists = set()
314
315 - def intersect(self, lists, name=None, description=None, tags=[]):
316 """Calculate the intersection of a given set of lists, and return the list representing the result""" 317 return self._do_operation(self.INTERSECTION_PATH, "Intersection", lists, name, description, tags)
318
319 - def union(self, lists, name=None, description=None, tags=[]):
320 """Calculate the union of a given set of lists, and return the list representing the result""" 321 return self._do_operation(self.UNION_PATH, "Union", lists, name, description, tags)
322
323 - def xor(self, lists, name=None, description=None, tags=[]):
324 """Calculate the symmetric difference of a given set of lists, and return the list representing the result""" 325 return self._do_operation(self.DIFFERENCE_PATH, "Difference", lists, name, description, tags)
326
327 - def subtract(self, lefts, rights, name=None, description=None, tags=[]):
328 """Calculate the subtraction of rights from lefts, and return the list representing the result""" 329 left_names = self.make_list_names(lefts) 330 right_names = self.make_list_names(rights) 331 if description is None: 332 description = "Subtraction of " + ' and '.join(right_names) + " from " + ' and '.join(left_names) 333 if name is None: 334 name = self.get_unused_list_name() 335 uri = self.service.root + self.SUBTRACTION_PATH 336 uri += '?' + urllib.urlencode({ 337 "name": name, 338 "description": description, 339 "references": ';'.join(left_names), 340 "subtract": ';'.join(right_names), 341 "tags": ";".join(tags) 342 }) 343 resp = self.service.opener.open(uri) 344 data = resp.read() 345 resp.close() 346 return self.parse_list_upload_response(data)
347
348 - def _do_operation(self, path, operation, lists, name, description, tags):
349 list_names = self.make_list_names(lists) 350 if description is None: 351 description = operation + " of " + ' and '.join(list_names) 352 if name is None: 353 name = self.get_unused_list_name() 354 uri = self.service.root + path 355 uri += '?' + urllib.urlencode({ 356 "name": name, 357 "lists": ';'.join(list_names), 358 "description": description, 359 "tags": ";".join(tags) 360 }) 361 resp = self.service.opener.open(uri) 362 data = resp.read() 363 resp.close() 364 return self.parse_list_upload_response(data)
365 366
367 - def make_list_names(self, lists):
368 """Turn a list of things into a list of list names""" 369 list_names = [] 370 for l in lists: 371 try: 372 t = l.list_type 373 list_names.append(l.name) 374 except AttributeError: 375 try: 376 m = l.model 377 list_names.append(self.create_list(l).name) 378 except AttributeError: 379 list_names.append(str(l)) 380 381 return list_names
382
383 -class ListServiceError(WebserviceError):
384 """Errors thrown when something goes wrong with list requests""" 385 pass
386