1 import weakref
2
3
4 try:
5 import json
6 except ImportError:
7 import simplejson as json
8
9 import urllib
10 import codecs
11
12 from intermine.errors import WebserviceError
13 from intermine.lists.list import List
16 """
17 A Class for Managing List Content and Operations
18 ================================================
19
20 This class provides methods to manage list contents and operations.
21
22 This class may be called itself, but all the useful methods it has
23 are also available on the Service object, which delegates to this class, while
24 other methods are more coneniently accessed through the list objects themselves.
25
26 NB: The methods for creating lists can conflict in threaded applications, if
27 two threads are each allocated the same unused list name. You are
28 strongly advised to use locks to synchronise any list creation requests (create_list,
29 or intersect, union, subtract, diff) unless you are choosing your own names each time
30 and are confident that these will not conflict.
31 """
32
33 DEFAULT_LIST_NAME = "my_list_"
34 DEFAULT_DESCRIPTION = "List created with Python client library"
35
36 INTERSECTION_PATH = '/lists/intersect/json'
37 UNION_PATH = '/lists/union/json'
38 DIFFERENCE_PATH = '/lists/diff/json'
39 SUBTRACTION_PATH = '/lists/subtract/json'
40
42 self.service = weakref.proxy(service)
43 self.lists = None
44 self._temp_lists = set()
45
47 """Update the list information with the latest details from the server"""
48 self.lists = {}
49 url = self.service.root + self.service.LIST_PATH
50 sock = self.service.opener.open(url)
51 data = sock.read()
52 sock.close()
53 list_info = json.loads(data)
54 if not list_info.get("wasSuccessful"):
55 raise ListServiceError(list_info.get("error"))
56 for l in list_info["lists"]:
57 l = ListManager.safe_dict(l)
58 self.lists[l["name"]] = List(service=self.service, manager=self, **l)
59
60 @staticmethod
62 """Recursively clone json structure with UTF-8 dictionary keys"""
63 if isinstance(d, dict):
64 return dict([(k.encode('utf-8'), v) for k,v in d.iteritems()])
65 else:
66 return d
67
69 """Return a list from the service by name, if it exists"""
70 if self.lists is None:
71 self.refresh_lists()
72 return self.lists.get(name)
73
77
83
85 """Get all the names of the lists in a particular webservice"""
86 if self.lists is None:
87 self.refresh_lists()
88 return self.lists.keys()
89
91 """
92 Return the number of lists accessible at the given webservice.
93 This number will vary depending on who you are authenticated as.
94 """
95 return len(self.get_all_list_names())
96
98 """
99 Get an unused list name
100 =======================
101
102 This method returns a new name that does not conflict
103 with any currently existing list name.
104
105 The list name is only guaranteed to be unused at the time
106 of allocation.
107 """
108 list_names = self.get_all_list_names()
109 counter = 1
110 name = self.DEFAULT_LIST_NAME + str(counter)
111 while name in list_names:
112 counter += 1
113 name = self.DEFAULT_LIST_NAME + str(counter)
114 self._temp_lists.add(name)
115 return name
116
118 q = queryable.to_query()
119 if not q.views:
120 q.add_view(q.root.name + ".id")
121 else:
122
123 up_to_attrs = set((v[0:v.rindex(".")] for v in q.views))
124 if len(up_to_attrs) == 1:
125 q.select(up_to_attrs.pop() + ".id")
126 return q
127
140
141 - def create_list(self, content, list_type="", name=None, description=None, tags=[], add=[]):
142 """
143 Create a new list in the webservice
144 ===================================
145
146 If no name is given, the list will be considered to be a temporary
147 list, and will be automatically deleted when the program ends. To prevent
148 this happening, give the list a name, either on creation, or by renaming it.
149
150 This method is not thread safe for anonymous lists - it will need synchronisation
151 with locks if you intend to create lists with multiple threads in parallel.
152
153 @param content: The source of the identifiers for this list. This can be:
154 * A string with white-space separated terms.
155 * The name of a file that contains the terms.
156 * A file-handle like thing (something with a 'read' method)
157 * An iterable of identifiers
158 * A query with a single column.
159 * Another list.
160 @param list_type: The type of objects to include in the list. This parameter is not
161 required if the content parameter implicitly includes the type
162 (as queries and lists do).
163 @param name: The name for the new list. If none is provided one will be generated, and the
164 list will be deleted when the list manager exits context.
165 @param description: A description for the list (free text, default = None)
166 @param tags: A set of strings to use as tags (default = [])
167 @param add: The issues groups that can be treated as matches. This should be a
168 collection of strings naming issue groups that would otherwise be ignored, but
169 in this case will be added to the list. The available groups are:
170 * DUPLICATE - More than one match was found.
171 * WILDCARD - A wildcard match was made.
172 * TYPE_CONVERTED - A match was found, but in another type (eg. found a protein
173 and we could convert it to a gene).
174 * OTHER - other issue types
175 * :all - All issues should be considered acceptable.
176 This only makes sense with text uploads - it is not required (or used) when
177 the content is a list or a query.
178
179 @rtype: intermine.lists.List
180 """
181 if description is None:
182 description = self.DEFAULT_DESCRIPTION
183
184 if name is None:
185 name = self.get_unused_list_name()
186
187 try:
188 ids = content.read()
189 except AttributeError:
190 try:
191 with open(content) as c:
192 ids = c.read()
193 except (TypeError, IOError):
194 try:
195 ids = content.strip()
196 except AttributeError:
197 try:
198 return self._create_list_from_queryable(content, name, description, tags)
199 except AttributeError:
200 try:
201 ids = "\n".join(map(lambda x: '"' + x + '"', iter(content)))
202 except AttributeError:
203 raise TypeError("Cannot create list from " + repr(content))
204
205 uri = self.service.root + self.service.LIST_CREATION_PATH
206 query_form = {
207 'name': name,
208 'type': list_type,
209 'description': description,
210 'tags': ";".join(tags)
211 }
212 if len(add): query_form['add'] = [x.lower() for x in add if x]
213
214 uri += "?" + urllib.urlencode(query_form, doseq = True)
215 data = self.service.opener.post_plain_text(uri, ids)
216 return self.parse_list_upload_response(data)
217
219 """
220 Intepret the response from the webserver to a list request, and return the List it describes
221 """
222 try:
223 response_data = json.loads(response)
224 except ValueError:
225 raise ListServiceError("Error parsing response: " + response)
226 if not response_data.get("wasSuccessful"):
227 raise ListServiceError(response_data.get("error"))
228 self.refresh_lists()
229 new_list = self.get_list(response_data["listName"])
230 failed_matches = response_data.get("unmatchedIdentifiers")
231 new_list._add_failed_matches(failed_matches)
232 return new_list
233
235 """Delete the given lists from the webserver"""
236 all_names = self.get_all_list_names()
237 for l in lists:
238 if isinstance(l, List):
239 name = l.name
240 else:
241 name = str(l)
242 if name not in all_names:
243 continue
244 uri = self.service.root + self.service.LIST_PATH
245 query_form = {'name': name}
246 uri += "?" + urllib.urlencode(query_form)
247 response = self.service.opener.delete(uri)
248 response_data = json.loads(response)
249 if not response_data.get("wasSuccessful"):
250 raise ListServiceError(response_data.get("error"))
251 self.refresh_lists()
252
265
279
294
295 - def _body_to_json(self, body):
296 try:
297 data = json.loads(body)
298 except ValueError:
299 raise ListServiceError("Error parsing response: " + body)
300 if not data.get("wasSuccessful"):
301 raise ListServiceError(data.get("error"))
302 return data
303
306
307 - def __exit__(self, exc_type, exc_val, traceback):
309
311 """Delete all the lists considered temporary (those created without names)"""
312 self.delete_lists(self._temp_lists)
313 self._temp_lists = set()
314
315 - def intersect(self, lists, name=None, description=None, tags=[]):
318
319 - def union(self, lists, name=None, description=None, tags=[]):
322
323 - def xor(self, lists, name=None, description=None, tags=[]):
326
327 - def subtract(self, lefts, rights, name=None, description=None, tags=[]):
328 """Calculate the subtraction of rights from lefts, and return the list representing the result"""
329 left_names = self.make_list_names(lefts)
330 right_names = self.make_list_names(rights)
331 if description is None:
332 description = "Subtraction of " + ' and '.join(right_names) + " from " + ' and '.join(left_names)
333 if name is None:
334 name = self.get_unused_list_name()
335 uri = self.service.root + self.SUBTRACTION_PATH
336 uri += '?' + urllib.urlencode({
337 "name": name,
338 "description": description,
339 "references": ';'.join(left_names),
340 "subtract": ';'.join(right_names),
341 "tags": ";".join(tags)
342 })
343 resp = self.service.opener.open(uri)
344 data = resp.read()
345 resp.close()
346 return self.parse_list_upload_response(data)
347
348 - def _do_operation(self, path, operation, lists, name, description, tags):
365
366
368 """Turn a list of things into a list of list names"""
369 list_names = []
370 for l in lists:
371 try:
372 t = l.list_type
373 list_names.append(l.name)
374 except AttributeError:
375 try:
376 m = l.model
377 list_names.append(self.create_list(l).name)
378 except AttributeError:
379 list_names.append(str(l))
380
381 return list_names
382
384 """Errors thrown when something goes wrong with list requests"""
385 pass
386