Package Gnumed :: Package pycommon :: Module gmPG2
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmPG2

   1  """GNUmed PostgreSQL connection handling. 
   2   
   3  TODO: iterator/generator batch fetching: 
   4          - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad 
   5          - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4" 
   6   
   7  winner: 
   8  def resultset_functional_batchgenerator(cursor, size=100): 
   9          for results in iter(lambda: cursor.fetchmany(size), []): 
  10                  for rec in results: 
  11                          yield rec 
  12  """ 
  13  # ======================================================================= 
  14  __version__ = "$Revision: 1.127 $" 
  15  __author__  = "K.Hilbert <Karsten.Hilbert@gmx.net>" 
  16  __license__ = 'GPL (details at http://www.gnu.org)' 
  17   
  18  ### imports ### 
  19  # stdlib 
  20  import time, locale, sys, re as regex, os, codecs, types, datetime as pydt, logging 
  21   
  22  # GNUmed 
  23  if __name__ == '__main__': 
  24          sys.path.insert(0, '../../') 
  25  from Gnumed.pycommon import gmLoginInfo, gmExceptions, gmDateTime, gmBorg, gmI18N, gmLog2 
  26   
  27  _log = logging.getLogger('gm.db') 
  28  _log.info(__version__) 
  29   
  30  # 3rd party 
  31  try: 
  32          import psycopg2 as dbapi 
  33  except ImportError: 
  34          _log.exception("Python database adapter psycopg2 not found.") 
  35          print "CRITICAL ERROR: Cannot find module psycopg2 for connecting to the database server." 
  36          raise 
  37  ### imports ### 
  38   
  39   
  40  _log.info('psycopg2 version: %s' % dbapi.__version__) 
  41  _log.info('PostgreSQL via DB-API module "%s": API level %s, thread safety %s, parameter style "%s"' % (dbapi, dbapi.apilevel, dbapi.threadsafety, dbapi.paramstyle)) 
  42  if not (float(dbapi.apilevel) >= 2.0): 
  43          raise ImportError('gmPG2: supported DB-API level too low') 
  44  if not (dbapi.threadsafety > 0): 
  45          raise ImportError('gmPG2: lacking minimum thread safety in psycopg2') 
  46  if not (dbapi.paramstyle == 'pyformat'): 
  47          raise ImportError('gmPG2: lacking pyformat (%%(<name>)s style) placeholder support in psycopg2') 
  48  try: 
  49          dbapi.__version__.index('dt') 
  50  except ValueError: 
  51          raise ImportError('gmPG2: lacking datetime support in psycopg2') 
  52  try: 
  53          dbapi.__version__.index('ext') 
  54  except ValueError: 
  55          raise ImportError('gmPG2: lacking extensions support in psycopg2') 
  56  try: 
  57          dbapi.__version__.index('pq3') 
  58  except ValueError: 
  59          raise ImportError('gmPG2: lacking v3 backend protocol support in psycopg2') 
  60   
  61  import psycopg2.extras 
  62  import psycopg2.extensions 
  63  import psycopg2.pool 
  64  import psycopg2.errorcodes as sql_error_codes 
  65   
  66  # ======================================================================= 
  67  _default_client_encoding = 'UTF8' 
  68  _log.info('assuming default client encoding of [%s]' % _default_client_encoding) 
  69   
  70  # things timezone 
  71  _default_client_timezone = None                 # default time zone for connections 
  72  _sql_set_timezone = None 
  73  _timestamp_template = "cast('%s' as timestamp with time zone)"          # MUST NOT be uniocde or else getquoted will not work 
  74  FixedOffsetTimezone = dbapi.tz.FixedOffsetTimezone 
  75   
  76  _default_dsn = None 
  77  _default_login = None 
  78   
  79  postgresql_version_string = None 
  80  postgresql_version = None                       # accuracy: major.minor 
  81   
  82  __ro_conn_pool = None 
  83   
  84  auto_request_login_params = True 
  85  # ======================================================================= 
  86  # global data 
  87  # ======================================================================= 
  88   
  89  known_schema_hashes = { 
  90          'devel': 'not released, testing only', 
  91          'v2': 'b09d50d7ed3f91ddf4c4ddb8ea507720', 
  92          'v3': 'e73718eaf230d8f1d2d01afa8462e176', 
  93          'v4': '4428ccf2e54c289136819e701bb095ea', 
  94          'v5': '7e7b093af57aea48c288e76632a382e5',       # ... old (v1) style hashes 
  95          'v6': '90e2026ac2efd236da9c8608b8685b2d',       # new (v2) style hashes ... 
  96          'v7': '6c9f6d3981483f8e9433df99d1947b27', 
  97          'v8': '89b13a7af83337c3aad153b717e52360', 
  98          'v9': '641a9b2be3c378ffc2bb2f0b1c9f051d', 
  99          'v10': '7ef42a8fb2bd929a2cdd0c63864b4e8a', 
 100          'v11': '03042ae24f3f92877d986fb0a6184d76', 
 101          'v12': '06183a6616db62257e22814007a8ed07', 
 102          'v13': 'fab7c1ae408a6530c47f9b5111a0841e', 
 103          'v14': 'e170d543f067d1ea60bfe9076b1560cf' 
 104  } 
 105   
 106  map_schema_hash2version = { 
 107          'b09d50d7ed3f91ddf4c4ddb8ea507720': 'v2', 
 108          'e73718eaf230d8f1d2d01afa8462e176': 'v3', 
 109          '4428ccf2e54c289136819e701bb095ea': 'v4', 
 110          '7e7b093af57aea48c288e76632a382e5': 'v5', 
 111          '90e2026ac2efd236da9c8608b8685b2d': 'v6', 
 112          '6c9f6d3981483f8e9433df99d1947b27': 'v7', 
 113          '89b13a7af83337c3aad153b717e52360': 'v8', 
 114          '641a9b2be3c378ffc2bb2f0b1c9f051d': 'v9', 
 115          '7ef42a8fb2bd929a2cdd0c63864b4e8a': 'v10', 
 116          '03042ae24f3f92877d986fb0a6184d76': 'v11', 
 117          '06183a6616db62257e22814007a8ed07': 'v12', 
 118          'fab7c1ae408a6530c47f9b5111a0841e': 'v13', 
 119          'e170d543f067d1ea60bfe9076b1560cf': 'v14' 
 120  } 
 121   
 122  map_client_branch2required_db_version = { 
 123          u'GIT tree': u'devel', 
 124          u'0.3': u'v9', 
 125          u'0.4': u'v10', 
 126          u'0.5': u'v11', 
 127          u'0.6': u'v12', 
 128          u'0.7': u'v13', 
 129          u'0.8': u'v14' 
 130  } 
 131   
 132  # get columns and data types for a given table 
 133  query_table_col_defs = u"""select 
 134          cols.column_name, 
 135          cols.udt_name 
 136  from 
 137          information_schema.columns cols 
 138  where 
 139          cols.table_schema = %s 
 140                  and 
 141          cols.table_name = %s 
 142  order by 
 143          cols.ordinal_position""" 
 144   
 145  query_table_attributes = u"""select 
 146          cols.column_name 
 147  from 
 148          information_schema.columns cols 
 149  where 
 150          cols.table_schema = %s 
 151                  and 
 152          cols.table_name = %s 
 153  order by 
 154          cols.ordinal_position""" 
 155   
 156  # ======================================================================= 
 157  # module globals API 
 158  # ======================================================================= 
159 -def set_default_client_encoding(encoding = None):
160 # check whether psycopg2 can handle this encoding 161 if encoding not in psycopg2.extensions.encodings: 162 raise ValueError('psycopg2 does not know how to handle client (wire) encoding [%s]' % encoding) 163 # check whether Python can handle this encoding 164 py_enc = psycopg2.extensions.encodings[encoding] 165 try: 166 codecs.lookup(py_enc) 167 except LookupError: 168 _log.warning('<codecs> module can NOT handle encoding [psycopg2::<%s> -> Python::<%s>]' % (encoding, py_enc)) 169 raise 170 # FIXME: check encoding against the database 171 # FIXME: - but we may not yet have access 172 # FIXME: - psycopg2 will pull its encodings from the database eventually 173 # it seems save to set it 174 global _default_client_encoding 175 _log.info('setting default client encoding from [%s] to [%s]' % (_default_client_encoding, str(encoding))) 176 _default_client_encoding = encoding 177 return True
178 #---------------------------------------------------
179 -def set_default_client_timezone(timezone = None):
180 181 # FIXME: use __validate 182 global _default_client_timezone 183 _log.info('setting default client time zone from [%s] to [%s]' % (_default_client_timezone, timezone)) 184 _default_client_timezone = timezone 185 186 global _sql_set_timezone 187 _sql_set_timezone = u'set timezone to %s' 188 189 return True
190 #---------------------------------------------------
191 -def __validate_timezone(conn=None, timezone=None):
192 193 _log.debug(u'validating time zone [%s]', timezone) 194 195 cmd = u'set timezone to %(tz)s' 196 args = {u'tz': timezone} 197 198 conn.commit() 199 curs = conn.cursor() 200 is_valid = False 201 try: 202 curs.execute(cmd, args) 203 _log.info(u'time zone [%s] is settable', timezone) 204 # can we actually use it, though ? 205 cmd = u"""select '1920-01-19 23:00:00+01'::timestamp with time zone""" 206 try: 207 curs.execute(cmd) 208 curs.fetchone() 209 _log.info(u'time zone [%s] is usable', timezone) 210 is_valid = True 211 except: 212 _log.error('error using time zone [%s]', timezone) 213 except dbapi.DataError: 214 _log.warning(u'time zone [%s] is not settable', timezone) 215 except: 216 _log.error(u'failed to set time zone to [%s]', timezone) 217 _log.exception(u'') 218 219 curs.close() 220 conn.rollback() 221 222 return is_valid
223 #---------------------------------------------------
224 -def __expand_timezone(conn=None, timezone=None):
225 """some timezone defs are abbreviations so try to expand 226 them because "set time zone" doesn't take abbreviations""" 227 228 cmd = u""" 229 select distinct on (abbrev) name 230 from pg_timezone_names 231 where 232 abbrev = %(tz)s and 233 name ~ '^[^/]+/[^/]+$' and 234 name !~ '^Etc/' 235 """ 236 args = {u'tz': timezone} 237 238 conn.commit() 239 curs = conn.cursor() 240 241 result = timezone 242 try: 243 curs.execute(cmd, args) 244 rows = curs.fetchall() 245 if len(rows) > 0: 246 result = rows[0][0] 247 _log.debug(u'[%s] maps to [%s]', timezone, result) 248 except: 249 _log.exception(u'cannot expand timezone abbreviation [%s]', timezone) 250 251 curs.close() 252 conn.rollback() 253 254 return result
255 #---------------------------------------------------
256 -def __detect_client_timezone(conn=None):
257 """This is run on the very first connection.""" 258 259 # FIXME: check whether server.timezone is the same 260 # FIXME: value as what we eventually detect 261 262 # we need gmDateTime to be initialized 263 if gmDateTime.current_local_iso_numeric_timezone_string is None: 264 gmDateTime.init() 265 266 _log.debug('trying to detect timezone from system') 267 268 tz_candidates = [] 269 try: 270 tz = os.environ['TZ'].decode(gmI18N.get_encoding(), 'replace') 271 tz_candidates.append(tz) 272 expanded = __expand_timezone(conn = conn, timezone = tz) 273 if expanded != tz: 274 tz_candidates.append(expanded) 275 except KeyError: 276 pass 277 278 tz_candidates.append(gmDateTime.current_local_timezone_name) 279 expanded = __expand_timezone(conn = conn, timezone = gmDateTime.current_local_timezone_name) 280 if expanded != gmDateTime.current_local_timezone_name: 281 tz_candidates.append(expanded) 282 283 _log.debug('candidates: %s', str(tz_candidates)) 284 285 # find best among candidates 286 global _default_client_timezone 287 global _sql_set_timezone 288 found = False 289 for tz in tz_candidates: 290 if __validate_timezone(conn = conn, timezone = tz): 291 _default_client_timezone = tz 292 _sql_set_timezone = u'set timezone to %s' 293 found = True 294 break 295 296 if not found: 297 _default_client_timezone = gmDateTime.current_local_iso_numeric_timezone_string 298 _sql_set_timezone = u"set time zone interval %s hour to minute" 299 300 _log.info('client system time zone detected as equivalent to [%s]', _default_client_timezone)
301 # ======================================================================= 302 # login API 303 # =======================================================================
304 -def __prompted_input(prompt, default=None):
305 usr_input = raw_input(prompt) 306 if usr_input == '': 307 return default 308 return usr_input
309 #---------------------------------------------------
310 -def __request_login_params_tui():
311 """Text mode request of database login parameters""" 312 import getpass 313 login = gmLoginInfo.LoginInfo() 314 315 print "\nPlease enter the required login parameters:" 316 try: 317 login.host = __prompted_input("host ['' = non-TCP/IP]: ", '') 318 login.database = __prompted_input("database [gnumed_v14]: ", 'gnumed_v14') 319 login.user = __prompted_input("user name: ", '') 320 tmp = 'password for "%s" (not shown): ' % login.user 321 login.password = getpass.getpass(tmp) 322 login.port = __prompted_input("port [5432]: ", 5432) 323 except KeyboardInterrupt: 324 _log.warning("user cancelled text mode login dialog") 325 print "user cancelled text mode login dialog" 326 raise gmExceptions.ConnectionError(_("Cannot connect to database without login information!")) 327 328 return login
329 #---------------------------------------------------
330 -def __request_login_params_gui_wx():
331 """GUI (wx) input request for database login parameters. 332 333 Returns gmLoginInfo.LoginInfo object 334 """ 335 import wx 336 # OK, wxPython was already loaded. But has the main Application instance 337 # been initialized yet ? if not, the exception will kick us out 338 if wx.GetApp() is None: 339 raise gmExceptions.NoGuiError(_("The wxPython GUI framework hasn't been initialized yet!")) 340 341 # Let's launch the login dialog 342 # if wx was not initialized /no main App loop, an exception should be raised anyway 343 import gmAuthWidgets 344 dlg = gmAuthWidgets.cLoginDialog(None, -1) 345 dlg.ShowModal() 346 login = dlg.panel.GetLoginInfo() 347 dlg.Destroy() 348 349 #if user cancelled or something else went wrong, raise an exception 350 if login is None: 351 raise gmExceptions.ConnectionError(_("Can't connect to database without login information!")) 352 353 return login
354 #---------------------------------------------------
355 -def request_login_params():
356 """Request login parameters for database connection.""" 357 # do we auto-request parameters at all ? 358 if not auto_request_login_params: 359 raise Exception('Cannot request login parameters.') 360 361 # are we inside X ? 362 # (if we aren't wxGTK will crash hard at 363 # C-level with "can't open Display") 364 if os.environ.has_key('DISPLAY'): 365 # try wxPython GUI 366 try: return __request_login_params_gui_wx() 367 except: pass 368 369 # well, either we are on the console or 370 # wxPython does not work, use text mode 371 return __request_login_params_tui()
372 373 # ======================================================================= 374 # DSN API 375 # -----------------------------------------------------------------------
376 -def make_psycopg2_dsn(database=None, host=None, port=5432, user=None, password=None):
377 dsn_parts = [] 378 379 if (database is not None) and (database.strip() != ''): 380 dsn_parts.append('dbname=%s' % database) 381 382 if (host is not None) and (host.strip() != ''): 383 dsn_parts.append('host=%s' % host) 384 385 if (port is not None) and (str(port).strip() != ''): 386 dsn_parts.append('port=%s' % port) 387 388 if (user is not None) and (user.strip() != ''): 389 dsn_parts.append('user=%s' % user) 390 391 if (password is not None) and (password.strip() != ''): 392 dsn_parts.append('password=%s' % password) 393 394 dsn_parts.append('sslmode=prefer') 395 396 return ' '.join(dsn_parts)
397 # ------------------------------------------------------
398 -def get_default_login():
399 # make sure we do have a login 400 get_default_dsn() 401 return _default_login
402 # ------------------------------------------------------
403 -def get_default_dsn():
404 global _default_dsn 405 if _default_dsn is not None: 406 return _default_dsn 407 408 login = request_login_params() 409 set_default_login(login=login) 410 411 return _default_dsn
412 # ------------------------------------------------------
413 -def set_default_login(login=None):
414 if login is None: 415 return False 416 417 if login.host is not None: 418 if login.host.strip() == u'': 419 login.host = None 420 421 global _default_login 422 _default_login = login 423 _log.info('setting default login from [%s] to [%s]' % (_default_login, login)) 424 425 dsn = make_psycopg2_dsn(login.database, login.host, login.port, login.user, login.password) 426 427 global _default_dsn 428 _default_dsn = dsn 429 _log.info('setting default DSN from [%s] to [%s]' % (_default_dsn, dsn)) 430 431 return True
432 # ======================================================================= 433 # netadata API 434 # =======================================================================
435 -def database_schema_compatible(link_obj=None, version=None, verbose=True):
436 expected_hash = known_schema_hashes[version] 437 if version == 'devel': 438 args = {'ver': '9999'} 439 else: 440 args = {'ver': version.strip('v')} 441 rows, idx = run_ro_queries ( 442 link_obj = link_obj, 443 queries = [{ 444 'cmd': u'select md5(gm.concat_table_structure(%(ver)s::integer)) as md5', 445 'args': args 446 }] 447 ) 448 if rows[0]['md5'] != expected_hash: 449 _log.error('database schema version mismatch') 450 _log.error('expected: %s (%s)' % (version, expected_hash)) 451 _log.error('detected: %s (%s)' % (get_schema_version(link_obj=link_obj), rows[0]['md5'])) 452 if verbose: 453 _log.debug('schema dump follows:') 454 for line in get_schema_structure(link_obj=link_obj).split(): 455 _log.debug(line) 456 _log.debug('schema revision history dump follows:') 457 for line in get_schema_revision_history(link_obj=link_obj): 458 _log.debug(u' - '.join(line)) 459 return False 460 _log.info('detected schema version [%s], hash [%s]' % (map_schema_hash2version[rows[0]['md5']], rows[0]['md5'])) 461 return True
462 #------------------------------------------------------------------------
463 -def get_schema_version(link_obj=None):
464 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': u'select md5(gm.concat_table_structure()) as md5'}]) 465 try: 466 return map_schema_hash2version[rows[0]['md5']] 467 except KeyError: 468 return u'unknown database schema version, MD5 hash is [%s]' % rows[0]['md5']
469 #------------------------------------------------------------------------
470 -def get_schema_structure(link_obj=None):
471 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': u'select gm.concat_table_structure()'}]) 472 return rows[0][0]
473 #------------------------------------------------------------------------
474 -def get_schema_revision_history(link_obj=None):
475 cmd = u""" 476 select 477 imported::text, 478 version, 479 filename 480 from gm.schema_revision 481 order by imported 482 """ 483 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': cmd}]) 484 return rows
485 #------------------------------------------------------------------------
486 -def get_current_user():
487 rows, idx = run_ro_queries(queries = [{'cmd': u'select CURRENT_USER'}]) 488 return rows[0][0]
489 #------------------------------------------------------------------------
490 -def get_foreign_keys2column(schema='public', table=None, column=None, link_obj=None):
491 """Get the foreign keys pointing to schema.table.column. 492 493 Does not properly work with multi-column FKs. 494 GNUmed doesn't use any, however. 495 """ 496 cmd = u""" 497 select 498 %(schema)s as referenced_schema, 499 %(tbl)s as referenced_table, 500 %(col)s as referenced_column, 501 pgc.confkey as referenced_column_list, 502 pgc.conrelid::regclass as referencing_table, 503 pgc.conkey as referencing_column_list, 504 (select attname from pg_attribute where attnum = pgc.conkey[1] and attrelid = pgc.conrelid) as referencing_column 505 from 506 pg_constraint pgc 507 where 508 pgc.contype = 'f' 509 and 510 pgc.confrelid = ( 511 select oid from pg_class where relname = %(tbl)s and relnamespace = ( 512 select oid from pg_namespace where nspname = %(schema)s 513 ) 514 ) and 515 ( 516 select attnum 517 from pg_attribute 518 where 519 attrelid = (select oid from pg_class where relname = %(tbl)s and relnamespace = ( 520 select oid from pg_namespace where nspname = %(schema)s 521 )) 522 and 523 attname = %(col)s 524 ) = any(pgc.confkey) 525 """ 526 527 args = { 528 'schema': schema, 529 'tbl': table, 530 'col': column 531 } 532 533 rows, idx = run_ro_queries ( 534 link_obj = link_obj, 535 queries = [ 536 {'cmd': cmd, 'args': args} 537 ] 538 ) 539 540 return rows
541 #------------------------------------------------------------------------
542 -def get_child_tables(schema='public', table=None, link_obj=None):
543 """Return child tables of <table>.""" 544 cmd = u""" 545 select 546 pgn.nspname as namespace, 547 pgc.relname as table 548 from 549 pg_namespace pgn, 550 pg_class pgc 551 where 552 pgc.relnamespace = pgn.oid 553 and 554 pgc.oid in ( 555 select inhrelid from pg_inherits where inhparent = ( 556 select oid from pg_class where 557 relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and 558 relname = %(table)s 559 ) 560 )""" 561 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': {'schema': schema, 'table': table}}]) 562 return rows
563 #------------------------------------------------------------------------
564 -def table_exists(link_obj=None, schema=None, table=None):
565 """Returns false, true.""" 566 cmd = u""" 567 select exists ( 568 select 1 from information_schema.tables 569 where 570 table_schema = %s and 571 table_name = %s and 572 table_type = 'BASE TABLE' 573 )""" 574 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': (schema, table)}]) 575 return rows[0][0]
576 #------------------------------------------------------------------------
577 -def get_col_indices(cursor = None):
578 if cursor.description is None: 579 _log.error('no result description available: unused cursor or last query did not select rows') 580 return None 581 col_indices = {} 582 col_index = 0 583 for col_desc in cursor.description: 584 col_name = col_desc[0] 585 # a query like "select 1,2;" will return two columns of the same name ! 586 # hence adjust to that, note, however, that dict-style access won't work 587 # on results of such queries ... 588 if col_indices.has_key(col_name): 589 col_name = '%s_%s' % (col_name, col_index) 590 col_indices[col_name] = col_index 591 col_index += 1 592 593 return col_indices
594 #------------------------------------------------------------------------
595 -def get_col_defs(link_obj=None, schema='public', table=None):
596 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_col_defs, 'args': (schema, table)}]) 597 col_names = [] 598 col_type = {} 599 for row in rows: 600 col_names.append(row[0]) 601 # map array types 602 if row[1].startswith('_'): 603 col_type[row[0]] = row[1][1:] + '[]' 604 else: 605 col_type[row[0]] = row[1] 606 col_defs = [] 607 col_defs.append(col_names) 608 col_defs.append(col_type) 609 return col_defs
610 #------------------------------------------------------------------------
611 -def get_col_names(link_obj=None, schema='public', table=None):
612 """Return column attributes of table""" 613 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_attributes, 'args': (schema, table)}]) 614 cols = [] 615 for row in rows: 616 cols.append(row[0]) 617 return cols
618 #------------------------------------------------------------------------
619 -def get_translation_languages():
620 rows, idx = run_ro_queries ( 621 queries = [{'cmd': u'select distinct lang from i18n.translations'}] 622 ) 623 return [ r[0] for r in rows ]
624 #------------------------------------------------------------------------
625 -def get_current_user_language():
626 cmd = u'select i18n.get_curr_lang()' 627 rows, idx = run_ro_queries(queries = [{'cmd': cmd}]) 628 return rows[0][0]
629 #------------------------------------------------------------------------
630 -def set_user_language(user=None, language=None):
631 """Set the user language in the database. 632 633 user = None: current db user 634 language = None: unset 635 """ 636 _log.info('setting database language for user [%s] to [%s]', user, language) 637 638 args = { 639 'usr': user, 640 'lang': language 641 } 642 643 if language is None: 644 if user is None: 645 queries = [{'cmd': u'select i18n.unset_curr_lang()'}] 646 else: 647 queries = [{'cmd': u'select i18n.unset_curr_lang(%(usr)s)', 'args': args}] 648 queries.append({'cmd': u'select True'}) 649 else: 650 if user is None: 651 queries = [{'cmd': u'select i18n.set_curr_lang(%(lang)s)', 'args': args}] 652 else: 653 queries = [{'cmd': u'select i18n.set_curr_lang(%(lang)s, %(usr)s)', 'args': args}] 654 655 rows, idx = run_rw_queries(queries = queries, return_data = True) 656 657 if not rows[0][0]: 658 _log.error('cannot set database language to [%s] for user [%s]', language, user) 659 660 return rows[0][0]
661 #------------------------------------------------------------------------
662 -def force_user_language(language=None):
663 """Set the user language in the database. 664 665 - regardless of whether there is any translation available. 666 - only for the current user 667 """ 668 _log.info('forcing database language for current db user to [%s]', language) 669 670 run_rw_queries(queries = [{ 671 'cmd': u'select i18n.force_curr_lang(%(lang)s)', 672 'args': {'lang': language} 673 }])
674 #------------------------------------------------------------------------ 675 #------------------------------------------------------------------------ 676 text_expansion_keywords = None 677
678 -def get_text_expansion_keywords():
679 global text_expansion_keywords 680 if text_expansion_keywords is not None: 681 return text_expansion_keywords 682 683 cmd = u"""select keyword, public_expansion, private_expansion, owner from clin.v_keyword_expansions""" 684 rows, idx = run_ro_queries(queries = [{'cmd': cmd}]) 685 text_expansion_keywords = rows 686 687 _log.info('retrieved %s text expansion keywords', len(text_expansion_keywords)) 688 689 return text_expansion_keywords
690 #------------------------------------------------------------------------
691 -def expand_keyword(keyword = None):
692 693 # Easter Egg ;-) 694 if keyword == u'$$steffi': 695 return u'Hai, play ! Versucht das ! (Keks dazu ?) :-)' 696 697 cmd = u"""select expansion from clin.v_your_keyword_expansions where keyword = %(kwd)s""" 698 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'kwd': keyword}}]) 699 700 if len(rows) == 0: 701 return None 702 703 return rows[0]['expansion']
704 #------------------------------------------------------------------------
705 -def get_keyword_expansion_candidates(keyword = None):
706 707 if keyword is None: 708 return [] 709 710 get_text_expansion_keywords() 711 712 candidates = [] 713 for kwd in text_expansion_keywords: 714 if kwd['keyword'].startswith(keyword): 715 candidates.append(kwd['keyword']) 716 717 return candidates
718 #------------------------------------------------------------------------
719 -def add_text_expansion(keyword=None, expansion=None, public=None):
720 721 if public: 722 cmd = u"select 1 from clin.v_keyword_expansions where public_expansion is true and keyword = %(kwd)s" 723 else: 724 cmd = u"select 1 from clin.v_your_keyword_expansions where private_expansion is true and keyword = %(kwd)s" 725 726 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'kwd': keyword}}]) 727 if len(rows) != 0: 728 return False 729 730 if public: 731 cmd = u""" 732 insert into clin.keyword_expansion (keyword, expansion, fk_staff) 733 values (%(kwd)s, %(exp)s, null)""" 734 else: 735 cmd = u""" 736 insert into clin.keyword_expansion (keyword, expansion, fk_staff) 737 values (%(kwd)s, %(exp)s, (select pk from dem.staff where db_user = current_user))""" 738 739 rows, idx = run_rw_queries(queries = [{'cmd': cmd, 'args': {'kwd': keyword, 'exp': expansion}}]) 740 741 global text_expansion_keywords 742 text_expansion_keywords = None 743 744 return True
745 #------------------------------------------------------------------------
746 -def delete_text_expansion(keyword):
747 cmd = u""" 748 delete from clin.keyword_expansion where 749 keyword = %(kwd)s and ( 750 (fk_staff = (select pk from dem.staff where db_user = current_user)) 751 or 752 (fk_staff is null and owner = current_user) 753 )""" 754 rows, idx = run_rw_queries(queries = [{'cmd': cmd, 'args': {'kwd': keyword}}]) 755 756 global text_expansion_keywords 757 text_expansion_keywords = None
758 #------------------------------------------------------------------------
759 -def edit_text_expansion(keyword, expansion):
760 761 cmd1 = u""" 762 delete from clin.keyword_expansion where 763 keyword = %(kwd)s and 764 fk_staff = (select pk from dem.staff where db_user = current_user)""" 765 766 cmd2 = u""" 767 insert into clin.keyword_expansion (keyword, expansion, fk_staff) 768 values (%(kwd)s, %(exp)s, (select pk from dem.staff where db_user = current_user))""" 769 770 rows, idx = run_rw_queries(queries = [ 771 {'cmd': cmd1, 'args': {'kwd': keyword}}, 772 {'cmd': cmd2, 'args': {'kwd': keyword, 'exp': expansion}}, 773 ]) 774 775 global text_expansion_keywords 776 text_expansion_keywords = None
777 # ======================================================================= 778 # query runners and helpers 779 # =======================================================================
780 -def send_maintenance_notification():
781 cmd = u'notify "db_maintenance_warning:"' 782 run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
783 #------------------------------------------------------------------------
784 -def send_maintenance_shutdown():
785 cmd = u'notify "db_maintenance_disconnect:"' 786 run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
787 #------------------------------------------------------------------------
788 -def is_pg_interval(candidate=None):
789 cmd = u'select %(candidate)s::interval' 790 try: 791 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}]) 792 return True 793 except: 794 cmd = u'select %(candidate)s::text::interval' 795 try: 796 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}]) 797 return True 798 except: 799 return False
800 #------------------------------------------------------------------------
801 -def bytea2file(data_query=None, filename=None, chunk_size=0, data_size=None, data_size_query=None):
802 outfile = file(filename, 'wb') 803 result = bytea2file_object(data_query=data_query, file_obj=outfile, chunk_size=chunk_size, data_size=data_size, data_size_query=data_size_query) 804 outfile.close() 805 return result
806 #------------------------------------------------------------------------
807 -def bytea2file_object(data_query=None, file_obj=None, chunk_size=0, data_size=None, data_size_query=None):
808 """Store data from a bytea field into a file. 809 810 <data_query> 811 - dict {'cmd': ..., 'args': ...} 812 - 'cmd' must be unicode containing "... substring(data from %(start)s for %(size)s) ..." 813 - 'args' must be a dict 814 - must return one row with one field of type bytea 815 <file> 816 - must be a file like Python object 817 <data_size> 818 - integer of the total size of the expected data or None 819 <data_size_query> 820 - dict {'cmd': ..., 'args': ...} 821 - cmd must be unicode 822 - must return one row with one field with the octet_length() of the data field 823 - used only when <data_size> is None 824 """ 825 if data_size == 0: 826 return True 827 828 # If the client sets an encoding other than the default we 829 # will receive encoding-parsed data which isn't the binary 830 # content we want. Hence we need to get our own connection. 831 # It must be a read-write one so that we don't affect the 832 # encoding for other users of the shared read-only 833 # connections. 834 # Actually, encodings shouldn't be applied to binary data 835 # (eg. bytea types) in the first place but that is only 836 # reported to be fixed > v7.4. 837 # further tests reveal that at least on PG 8.0 this bug still 838 # manifests itself 839 conn = get_raw_connection(readonly=True) 840 841 if data_size is None: 842 rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query]) 843 data_size = rows[0][0] 844 if data_size in [None, 0]: 845 conn.rollback() 846 return True 847 848 _log.debug('expecting bytea data of size: [%s] bytes' % data_size) 849 _log.debug('using chunk size of: [%s] bytes' % chunk_size) 850 851 # chunk size of 0 means "retrieve whole field at once" 852 if chunk_size == 0: 853 chunk_size = data_size 854 _log.debug('chunk size [0] bytes: retrieving all data at once') 855 856 # Windoze sucks: it can't transfer objects of arbitrary size, 857 # anyways, we need to split the transfer, 858 # however, only possible if postgres >= 7.2 859 needed_chunks, remainder = divmod(data_size, chunk_size) 860 _log.debug('chunks to retrieve: [%s]' % needed_chunks) 861 _log.debug('remainder to retrieve: [%s] bytes' % remainder) 862 863 # retrieve chunks, skipped if data size < chunk size, 864 # does this not carry the danger of cutting up multi-byte escape sequences ? 865 # no, since bytea is binary, 866 # yes, since in bytea there are *some* escaped values, still 867 # no, since those are only escaped during *transfer*, not on-disk, hence 868 # only complete escape sequences are put on the wire 869 for chunk_id in range(needed_chunks): 870 chunk_start = (chunk_id * chunk_size) + 1 871 data_query['args']['start'] = chunk_start 872 data_query['args']['size'] = chunk_size 873 try: 874 rows, idx = run_ro_queries(link_obj=conn, queries=[data_query]) 875 except: 876 _log.error('cannot retrieve chunk [%s/%s], size [%s], try decreasing chunk size' % (chunk_id+1, needed_chunks, chunk_size)) 877 conn.rollback() 878 raise 879 # it would be a fatal error to see more than one result as ids are supposed to be unique 880 file_obj.write(str(rows[0][0])) 881 882 # retrieve remainder 883 if remainder > 0: 884 chunk_start = (needed_chunks * chunk_size) + 1 885 data_query['args']['start'] = chunk_start 886 data_query['args']['size'] = remainder 887 try: 888 rows, idx = run_ro_queries(link_obj=conn, queries=[data_query]) 889 except: 890 _log.error('cannot retrieve remaining [%s] bytes' % remainder) 891 conn.rollback() 892 raise 893 # it would be a fatal error to see more than one result as ids are supposed to be unique 894 file_obj.write(str(rows[0][0])) 895 896 conn.rollback() 897 return True
898 #------------------------------------------------------------------------
899 -def file2bytea(query=None, filename=None, args=None, conn=None):
900 """Store data from a file into a bytea field. 901 902 The query must: 903 - be in unicode 904 - contain a format spec identifying the row (eg a primary key) 905 matching <args> if it is an UPDATE 906 - contain a format spec %(data)s::bytea 907 """ 908 # read data from file 909 infile = file(filename, "rb") 910 data_as_byte_string = infile.read() 911 infile.close() 912 if args is None: 913 args = {} 914 args['data'] = buffer(data_as_byte_string) 915 del(data_as_byte_string) 916 917 # insert the data 918 if conn is None: 919 conn = get_raw_connection(readonly=False) 920 close_conn = True 921 else: 922 close_conn = False 923 924 run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = True) 925 926 if close_conn: 927 conn.close() 928 929 return
930 #------------------------------------------------------------------------
931 -def sanitize_pg_regex(expression=None, escape_all=False):
932 """Escape input for use in a PostgreSQL regular expression. 933 934 If a fragment comes from user input and is to be used 935 as a regular expression we need to make sure it doesn't 936 contain invalid regex patterns such as unbalanced ('s. 937 938 <escape_all> 939 True: try to escape *all* metacharacters 940 False: only escape those which render the regex invalid 941 """ 942 return expression.replace ( 943 '(', '\(' 944 ).replace ( 945 ')', '\)' 946 ).replace ( 947 '[', '\[' 948 ).replace ( 949 '+', '\+' 950 ).replace ( 951 '.', '\.' 952 ).replace ( 953 '*', '\*' 954 )
955 #']', '\]', # not needed 956 #------------------------------------------------------------------------
957 -def run_ro_queries(link_obj=None, queries=None, verbose=False, return_data=True, get_col_idx=False):
958 """Run read-only queries. 959 960 <queries> must be a list of dicts: 961 [ 962 {'cmd': <string>, 'args': <dict> or <tuple>}, 963 {...}, 964 ... 965 ] 966 """ 967 if isinstance(link_obj, dbapi._psycopg.cursor): 968 curs = link_obj 969 curs_close = __noop 970 tx_rollback = __noop 971 elif isinstance(link_obj, dbapi._psycopg.connection): 972 curs = link_obj.cursor() 973 curs_close = curs.close 974 tx_rollback = link_obj.rollback 975 elif link_obj is None: 976 conn = get_connection(readonly=True, verbose=verbose) 977 curs = conn.cursor() 978 curs_close = curs.close 979 tx_rollback = conn.rollback 980 else: 981 raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj) 982 983 if verbose: 984 _log.debug('cursor: %s', curs) 985 986 for query in queries: 987 if type(query['cmd']) is not types.UnicodeType: 988 print "run_ro_queries(): non-unicode query" 989 print query['cmd'] 990 try: 991 args = query['args'] 992 except KeyError: 993 args = None 994 try: 995 curs.execute(query['cmd'], args) 996 if verbose: 997 _log.debug('ran query: [%s]', curs.query) 998 _log.debug('PG status message: %s', curs.statusmessage) 999 _log.debug('cursor description: %s', str(curs.description)) 1000 except: 1001 # FIXME: use .pgcode 1002 try: 1003 curs_close() 1004 except dbapi.InterfaceError: 1005 _log.exception('cannot close cursor') 1006 tx_rollback() # need to rollback so ABORT state isn't preserved in pooled conns 1007 _log.error('query failed: [%s]', curs.query) 1008 _log.error('PG status message: %s', curs.statusmessage) 1009 raise 1010 1011 data = None 1012 col_idx = None 1013 if return_data: 1014 data = curs.fetchall() 1015 if verbose: 1016 _log.debug('last query returned [%s (%s)] rows', curs.rowcount, len(data)) 1017 _log.debug('cursor description: %s', str(curs.description)) 1018 if get_col_idx: 1019 col_idx = get_col_indices(curs) 1020 1021 curs_close() 1022 tx_rollback() # rollback just so that we don't stay IDLE IN TRANSACTION forever 1023 return (data, col_idx)
1024 #------------------------------------------------------------------------
1025 -def run_rw_queries(link_obj=None, queries=None, end_tx=False, return_data=None, get_col_idx=False, verbose=False):
1026 """Convenience function for running a transaction 1027 that is supposed to get committed. 1028 1029 <link_obj> 1030 can be either: 1031 - a cursor 1032 - a connection 1033 1034 <queries> 1035 is a list of dicts [{'cmd': <string>, 'args': <dict> or <tuple>) 1036 to be executed as a single transaction, the last 1037 query may usefully return rows (such as a 1038 "select currval('some_sequence')" statement) 1039 1040 <end_tx> 1041 - controls whether the transaction is finalized (eg. 1042 committed/rolled back) or not, this allows the 1043 call to run_rw_queries() to be part of a framing 1044 transaction 1045 - if link_obj is a connection then <end_tx> will 1046 default to False unless it is explicitly set to 1047 True which is taken to mean "yes, you do have full 1048 control over the transaction" in which case the 1049 transaction is properly finalized 1050 - if link_obj is a cursor we CANNOT finalize the 1051 transaction because we would need the connection for that 1052 - if link_obj is None <end_tx> will, of course, always be True 1053 1054 <return_data> 1055 - if true, the returned data will include the rows 1056 the last query selected 1057 - if false, it returns None instead 1058 1059 <get_col_idx> 1060 - if true, the returned data will include a dictionary 1061 mapping field names to column positions 1062 - if false, the returned data returns None instead 1063 1064 method result: 1065 - returns a tuple (data, idx) 1066 - <data>: 1067 * (None, None) if last query did not return rows 1068 * ("fetchall() result", <index>) if last query returned any rows 1069 * for <index> see <get_col_idx> 1070 """ 1071 if isinstance(link_obj, dbapi._psycopg.cursor): 1072 conn_close = __noop 1073 conn_commit = __noop 1074 conn_rollback = __noop 1075 curs = link_obj 1076 curs_close = __noop 1077 elif isinstance(link_obj, dbapi._psycopg.connection): 1078 conn_close = __noop 1079 if end_tx: 1080 conn_commit = link_obj.commit 1081 conn_rollback = link_obj.rollback 1082 else: 1083 conn_commit = __noop 1084 conn_rollback = __noop 1085 curs = link_obj.cursor() 1086 curs_close = curs.close 1087 elif link_obj is None: 1088 conn = get_connection(readonly=False) 1089 conn_close = conn.close 1090 conn_commit = conn.commit 1091 conn_rollback = conn.rollback 1092 curs = conn.cursor() 1093 curs_close = curs.close 1094 else: 1095 raise ValueError('link_obj must be cursor, connection or None and not [%s]' % link_obj) 1096 1097 for query in queries: 1098 if type(query['cmd']) is not types.UnicodeType: 1099 print "run_rw_queries(): non-unicode query" 1100 print query['cmd'] 1101 try: 1102 args = query['args'] 1103 except KeyError: 1104 args = None 1105 try: 1106 curs.execute(query['cmd'], args) 1107 except: 1108 _log.exception('error running RW query') 1109 gmLog2.log_stack_trace() 1110 try: 1111 curs_close() 1112 conn_rollback() 1113 conn_close() 1114 except dbapi.InterfaceError: 1115 _log.exception('cannot cleanup') 1116 raise 1117 raise 1118 1119 data = None 1120 col_idx = None 1121 if return_data: 1122 try: 1123 data = curs.fetchall() 1124 except: 1125 _log.exception('error fetching data from RW query') 1126 gmLog2.log_stack_trace() 1127 try: 1128 curs_close() 1129 conn_rollback() 1130 conn_close() 1131 except dbapi.InterfaceError: 1132 _log.exception('cannot cleanup') 1133 raise 1134 raise 1135 if get_col_idx: 1136 col_idx = get_col_indices(curs) 1137 1138 curs_close() 1139 conn_commit() 1140 conn_close() 1141 1142 return (data, col_idx)
1143 # ======================================================================= 1144 # connection handling API 1145 # -----------------------------------------------------------------------
1146 -class cConnectionPool(psycopg2.pool.PersistentConnectionPool):
1147 """ 1148 GNUmed database connection pool. 1149 1150 Extends psycopg2's PersistentConnectionPool with 1151 a custom _connect() function. Supports one connection 1152 per thread - which also ties it to one particular DSN. 1153 """ 1154 #--------------------------------------------------
1155 - def _connect(self, key=None):
1156 1157 conn = get_raw_connection(dsn = self._kwargs['dsn'], verbose = self._kwargs['verbose'], readonly=True) 1158 1159 conn.original_close = conn.close 1160 conn.close = _raise_exception_on_ro_conn_close 1161 1162 if key is not None: 1163 self._used[key] = conn 1164 self._rused[id(conn)] = key 1165 else: 1166 self._pool.append(conn) 1167 1168 return conn
1169 #--------------------------------------------------
1170 - def shutdown(self):
1171 for conn_key in self._used.keys(): 1172 _log.debug('closing pooled database connection, pool key: %s, backend PID: %s', conn_key, self._used[conn_key].get_backend_pid()) 1173 self._used[conn_key].original_close()
1174 # -----------------------------------------------------------------------
1175 -def get_raw_connection(dsn=None, verbose=False, readonly=True):
1176 """Get a raw, unadorned connection. 1177 1178 - this will not set any parameters such as encoding, timezone, datestyle 1179 - the only requirement is a valid DSN 1180 - hence it can be used for "service" connections 1181 for verifying encodings etc 1182 """ 1183 # FIXME: support verbose 1184 if dsn is None: 1185 dsn = get_default_dsn() 1186 1187 try: 1188 conn = dbapi.connect(dsn=dsn, connection_factory=psycopg2.extras.DictConnection) 1189 except dbapi.OperationalError, e: 1190 1191 t, v, tb = sys.exc_info() 1192 try: 1193 msg = e.args[0] 1194 except (AttributeError, IndexError, TypeError): 1195 raise 1196 1197 msg = unicode(msg, gmI18N.get_encoding(), 'replace') 1198 1199 if msg.find('fe_sendauth') != -1: 1200 raise cAuthenticationError, (dsn, msg), tb 1201 1202 if regex.search('user ".*" does not exist', msg) is not None: 1203 raise cAuthenticationError, (dsn, msg), tb 1204 1205 if msg.find('uthenti') != -1: 1206 raise cAuthenticationError, (dsn, msg), tb 1207 1208 raise 1209 1210 _log.debug('new database connection, backend PID: %s, readonly: %s', conn.get_backend_pid(), readonly) 1211 1212 # do first-time stuff 1213 global postgresql_version 1214 if postgresql_version is None: 1215 curs = conn.cursor() 1216 curs.execute (""" 1217 select 1218 (split_part(setting, '.', 1) || '.' || split_part(setting, '.', 2))::numeric as version 1219 from pg_settings 1220 where name='server_version'""" 1221 ) 1222 postgresql_version = curs.fetchone()['version'] 1223 _log.info('PostgreSQL version (numeric): %s' % postgresql_version) 1224 try: 1225 curs.execute("select pg_size_pretty(pg_database_size(current_database()))") 1226 _log.info('database size: %s', curs.fetchone()[0]) 1227 except: 1228 pass 1229 if verbose: 1230 __log_PG_settings(curs=curs) 1231 curs.close() 1232 conn.commit() 1233 1234 if _default_client_timezone is None: 1235 __detect_client_timezone(conn = conn) 1236 1237 curs = conn.cursor() 1238 1239 # set access mode 1240 if readonly: 1241 _log.debug('access mode [READ ONLY]') 1242 cmd = 'set session characteristics as transaction READ ONLY' 1243 curs.execute(cmd) 1244 cmd = 'set default_transaction_read_only to on' 1245 curs.execute(cmd) 1246 else: 1247 _log.debug('access mode [READ WRITE]') 1248 cmd = 'set session characteristics as transaction READ WRITE' 1249 curs.execute(cmd) 1250 cmd = 'set default_transaction_read_only to off' 1251 curs.execute(cmd) 1252 1253 curs.close() 1254 conn.commit() 1255 1256 conn.is_decorated = False 1257 1258 return conn
1259 # =======================================================================
1260 -def get_connection(dsn=None, readonly=True, encoding=None, verbose=False, pooled=True):
1261 """Get a new connection. 1262 1263 This assumes the locale system has been initialized 1264 unless an encoding is specified. 1265 """ 1266 # FIXME: support pooled on RW, too 1267 # FIXME: for now, support the default DSN only 1268 if pooled and readonly and (dsn is None): 1269 global __ro_conn_pool 1270 if __ro_conn_pool is None: 1271 __ro_conn_pool = cConnectionPool ( 1272 minconn = 1, 1273 maxconn = 2, 1274 dsn = dsn, 1275 verbose = verbose 1276 ) 1277 conn = __ro_conn_pool.getconn() 1278 else: 1279 conn = get_raw_connection(dsn=dsn, verbose=verbose, readonly=False) 1280 1281 if conn.is_decorated: 1282 return conn 1283 1284 if encoding is None: 1285 encoding = _default_client_encoding 1286 if encoding is None: 1287 encoding = gmI18N.get_encoding() 1288 _log.warning('client encoding not specified') 1289 _log.warning('the string encoding currently set in the active locale is used: [%s]' % encoding) 1290 _log.warning('for this to work properly the application MUST have called locale.setlocale() before') 1291 1292 # set connection properties 1293 # 1) client encoding 1294 try: 1295 conn.set_client_encoding(encoding) 1296 except dbapi.OperationalError: 1297 t, v, tb = sys.exc_info() 1298 if str(v).find("can't set encoding to") != -1: 1299 raise cEncodingError, (encoding, v), tb 1300 raise 1301 1302 # 2) transaction isolation level 1303 if readonly: 1304 conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) 1305 iso_level = u'read committed' 1306 else: 1307 conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) 1308 iso_level = u'serializable' 1309 1310 _log.debug('client string encoding [%s], isolation level [%s], time zone [%s], datestyle [ISO], sql_inheritance [ON]', encoding, iso_level, _default_client_timezone) 1311 1312 curs = conn.cursor() 1313 1314 # client time zone 1315 curs.execute(_sql_set_timezone, [_default_client_timezone]) 1316 1317 # datestyle 1318 # regarding DMY/YMD handling: since we force *input* to 1319 # ISO, too, the DMY/YMD setting is not needed 1320 cmd = "set datestyle to 'ISO'" 1321 curs.execute(cmd) 1322 1323 # SQL inheritance mode 1324 cmd = 'set sql_inheritance to on' 1325 curs.execute(cmd) 1326 1327 # version string 1328 global postgresql_version_string 1329 if postgresql_version_string is None: 1330 curs.execute('select version()') 1331 postgresql_version_string = curs.fetchone()['version'] 1332 _log.info('PostgreSQL version (string): "%s"' % postgresql_version_string) 1333 1334 curs.close() 1335 conn.commit() 1336 1337 conn.is_decorated = True 1338 1339 return conn
1340 #-----------------------------------------------------------------------
1341 -def shutdown():
1342 if __ro_conn_pool is None: 1343 return 1344 __ro_conn_pool.shutdown()
1345 # ====================================================================== 1346 # internal helpers 1347 #-----------------------------------------------------------------------
1348 -def __noop():
1349 pass
1350 #-----------------------------------------------------------------------
1351 -def _raise_exception_on_ro_conn_close():
1352 raise TypeError(u'close() called on read-only connection')
1353 #-----------------------------------------------------------------------
1354 -def sanity_check_time_skew(tolerance=60):
1355 """Check server time and local time to be within 1356 the given tolerance of each other. 1357 1358 tolerance: seconds 1359 """ 1360 _log.debug('maximum skew tolerance (seconds): %s', tolerance) 1361 1362 cmd = u"select now() at time zone 'UTC'" 1363 conn = get_raw_connection(readonly=True) 1364 curs = conn.cursor() 1365 1366 start = time.time() 1367 rows, idx = run_ro_queries(link_obj = curs, queries = [{'cmd': cmd}]) 1368 end = time.time() 1369 client_now_as_utc = pydt.datetime.utcnow() 1370 1371 curs.close() 1372 conn.commit() 1373 1374 server_now_as_utc = rows[0][0] 1375 query_duration = end - start 1376 _log.info('server "now" (UTC): %s', server_now_as_utc) 1377 _log.info('client "now" (UTC): %s', client_now_as_utc) 1378 _log.debug('wire roundtrip (seconds): %s', query_duration) 1379 1380 if query_duration > tolerance: 1381 _log.error('useless to check client/server time skew, wire roundtrip > tolerance') 1382 return False 1383 1384 if server_now_as_utc > client_now_as_utc: 1385 real_skew = server_now_as_utc - client_now_as_utc 1386 else: 1387 real_skew = client_now_as_utc - server_now_as_utc 1388 1389 _log.debug('client/server time skew: %s', real_skew) 1390 1391 if real_skew > pydt.timedelta(seconds = tolerance): 1392 _log.error('client/server time skew > tolerance') 1393 return False 1394 1395 return True
1396 #-----------------------------------------------------------------------
1397 -def sanity_check_database_settings():
1398 """Checks database settings. 1399 1400 returns (status, message) 1401 status: 1402 0: no problem 1403 1: non-fatal problem 1404 2: fatal problem 1405 """ 1406 _log.debug('checking database settings') 1407 settings = { 1408 # setting: [expected value, risk, fatal?] 1409 u'allow_system_table_mods': [u'off', u'system breakage', False], 1410 u'check_function_bodies': [u'on', u'suboptimal error detection', False], 1411 u'default_transaction_read_only': [u'on', u'accidental database writes', False], 1412 u'fsync': [u'on', u'data loss/corruption', True], 1413 u'full_page_writes': [u'on', u'data loss/corruption', False], 1414 u'lc_messages': [u'C', u'suboptimal error detection', False], 1415 u'password_encryption': [u'on', u'breach of confidentiality', False], 1416 u'regex_flavor': [u'advanced', u'query breakage', False], 1417 u'synchronous_commit': [u'on', u'data loss/corruption', False], 1418 u'sql_inheritance': [u'on', u'query breakage, data loss/corruption', True] 1419 } 1420 1421 from Gnumed.pycommon import gmCfg2 1422 _cfg = gmCfg2.gmCfgData() 1423 if _cfg.get(option = u'hipaa'): 1424 settings[u'log_connections'] = [u'on', u'non-compliance with HIPAA', True] 1425 settings[u'log_disconnections'] = [u'on', u'non-compliance with HIPAA', True] 1426 else: 1427 settings[u'log_connections'] = [u'on', u'non-compliance with HIPAA', None] 1428 settings[u'log_disconnections'] = [u'on', u'non-compliance with HIPAA', None] 1429 1430 cmd = u"select name, setting from pg_settings where name in %(settings)s" 1431 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'settings': tuple(settings.keys())}}]) 1432 1433 found_error = False 1434 found_problem = False 1435 msg = [] 1436 for row in rows: 1437 if row[1] != settings[row[0]][0]: 1438 if settings[row[0]][2] is True: 1439 found_error = True 1440 elif settings[row[0]][2] is False: 1441 found_problem = True 1442 elif settings[row[0]][2] is None: 1443 pass 1444 else: 1445 _log.error(settings[row[0]]) 1446 raise ValueError(u'invalid database configuration sanity check') 1447 msg.append(_(' option [%s]: %s') % (row[0], row[1])) 1448 msg.append(_(' risk: %s') % settings[row[0]][1]) 1449 _log.warning('PG option [%s] set to [%s], expected [%s], risk: <%s>' % (row[0], row[1], settings[row[0]][0], settings[row[0]][1])) 1450 1451 if found_error: 1452 return 2, u'\n'.join(msg) 1453 1454 if found_problem: 1455 return 1, u'\n'.join(msg) 1456 1457 return 0, u''
1458 #------------------------------------------------------------------------
1459 -def __log_PG_settings(curs=None):
1460 # don't use any of the run_*()s since that might 1461 # create a loop if we fail here 1462 # FIXME: use pg_settings 1463 try: 1464 curs.execute(u'show all') 1465 except: 1466 _log.exception(u'cannot log PG settings (>>>show all<<< failed)') 1467 return False 1468 settings = curs.fetchall() 1469 if settings is None: 1470 _log.error(u'cannot log PG settings (>>>show all<<< did not return rows)') 1471 return False 1472 for setting in settings: 1473 _log.debug(u'PG option [%s]: %s', setting[0], setting[1]) 1474 return True
1475 # =======================================================================
1476 -def extract_msg_from_pg_exception(exc=None):
1477 1478 try: 1479 msg = exc.args[0] 1480 except (AttributeError, IndexError, TypeError): 1481 return u'cannot extract message from exception' 1482 1483 return unicode(msg, gmI18N.get_encoding(), 'replace')
1484 # =======================================================================
1485 -class cAuthenticationError(dbapi.OperationalError):
1486
1487 - def __init__(self, dsn=None, prev_val=None):
1488 self.dsn = dsn 1489 self.prev_val = prev_val
1490
1491 - def __str__(self):
1492 _log.warning('%s.__str__() called', self.__class__.__name__) 1493 tmp = u'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn) 1494 _log.error(tmp) 1495 return tmp.encode(gmI18N.get_encoding(), 'replace')
1496
1497 - def __unicode__(self):
1498 return u'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn)
1499 1500 # ======================================================================= 1501 # custom psycopg2 extensions 1502 # =======================================================================
1503 -class cEncodingError(dbapi.OperationalError):
1504
1505 - def __init__(self, encoding=None, prev_val=None):
1506 self.encoding = encoding 1507 self.prev_val = prev_val
1508
1509 - def __str__(self):
1510 _log.warning('%s.__str__() called', self.__class__.__name__) 1511 return 'PostgreSQL: %s\nencoding: %s' % (self.prev_val.encode(gmI18N.get_encoding(), 'replace'), self.encoding.encode(gmI18N.get_encoding(), 'replace'))
1512
1513 - def __unicode__(self):
1514 return u'PostgreSQL: %s\nencoding: %s' % (self.prev_val, self.encoding)
1515 1516 # ----------------------------------------------------------------------- 1517 # Python -> PostgreSQL 1518 # ----------------------------------------------------------------------- 1519 # test when Squeeze (and thus psycopg2 2.2 becomes Stable
1520 -class cAdapterPyDateTime(object):
1521
1522 - def __init__(self, dt):
1523 if dt.tzinfo is None: 1524 raise ValueError(u'datetime.datetime instance is lacking a time zone: [%s]' % _timestamp_template % dt.isoformat()) 1525 self.__dt = dt
1526
1527 - def getquoted(self):
1528 return _timestamp_template % self.__dt.isoformat()
1529 1530 # remove for 0.9 1531 # ---------------------------------------------------------------------- 1532 #class cAdapterMxDateTime(object): 1533 # 1534 # def __init__(self, dt): 1535 # if dt.tz == '???': 1536 # _log.info('[%s]: no time zone string available in (%s), assuming local time zone', self.__class__.__name__, dt) 1537 # self.__dt = dt 1538 # 1539 # def getquoted(self): 1540 # # under some locale settings the mx.DateTime ISO formatter 1541 # # will insert "," into the ISO string, 1542 # # while this is allowed per the ISO8601 spec PostgreSQL 1543 # # cannot currently handle that, 1544 # # so map those "," to "." to make things work: 1545 # return mxDT.ISO.str(self.__dt).replace(',', '.') 1546 # 1547 # ---------------------------------------------------------------------- 1548 # PostgreSQL -> Python 1549 # ---------------------------------------------------------------------- 1550 1551 #======================================================================= 1552 # main 1553 #----------------------------------------------------------------------- 1554 1555 # make sure psycopg2 knows how to handle unicode ... 1556 # intended to become standard 1557 # test when Squeeze (and thus psycopg2 2.2 becomes Stable 1558 psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) 1559 psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY) 1560 1561 # tell psycopg2 how to adapt datetime types with timestamps when locales are in use 1562 # check in 0.9: 1563 psycopg2.extensions.register_adapter(pydt.datetime, cAdapterPyDateTime) 1564 1565 # remove for 0.9 1566 try: 1567 import mx.DateTime as mxDT 1568 # psycopg2.extensions.register_adapter(mxDT.DateTimeType, cAdapterMxDateTime) 1569 except ImportError: 1570 _log.warning('cannot import mx.DateTime') 1571 1572 # do NOT adapt *lists* to "... IN (*) ..." syntax because we want 1573 # them adapted to "... ARRAY()..." so we can support PG arrays 1574 1575 #======================================================================= 1576 if __name__ == "__main__": 1577 1578 if len(sys.argv) < 2: 1579 sys.exit() 1580 1581 if sys.argv[1] != 'test': 1582 sys.exit() 1583 1584 logging.basicConfig(level=logging.DEBUG) 1585 #--------------------------------------------------------------------
1586 - def test_file2bytea():
1587 run_rw_queries(queries = [ 1588 {'cmd': u'create table test_bytea (data bytea)'} 1589 ]) 1590 1591 cmd = u'insert into test_bytea values (%(data)s::bytea)' 1592 try: 1593 file2bytea(query = cmd, filename = sys.argv[2]) 1594 except: 1595 _log.exception('error') 1596 1597 run_rw_queries(queries = [ 1598 {'cmd': u'drop table test_bytea'} 1599 ])
1600 #--------------------------------------------------------------------
1601 - def test_get_connection():
1602 print "testing get_connection()" 1603 1604 dsn = 'foo' 1605 try: 1606 conn = get_connection(dsn=dsn) 1607 except dbapi.OperationalError, e: 1608 print "SUCCESS: get_connection(%s) failed as expected" % dsn 1609 t, v = sys.exc_info()[:2] 1610 print ' ', t 1611 print ' ', v 1612 1613 dsn = 'dbname=gnumed_v9' 1614 try: 1615 conn = get_connection(dsn=dsn) 1616 except cAuthenticationError: 1617 print "SUCCESS: get_connection(%s) failed as expected" % dsn 1618 t, v = sys.exc_info()[:2] 1619 print ' ', t 1620 print ' ', v 1621 1622 dsn = 'dbname=gnumed_v9 user=abc' 1623 try: 1624 conn = get_connection(dsn=dsn) 1625 except cAuthenticationError: 1626 print "SUCCESS: get_connection(%s) failed as expected" % dsn 1627 t, v = sys.exc_info()[:2] 1628 print ' ', t 1629 print ' ', v 1630 1631 dsn = 'dbname=gnumed_v9 user=any-doc' 1632 try: 1633 conn = get_connection(dsn=dsn) 1634 except cAuthenticationError: 1635 print "SUCCESS: get_connection(%s) failed as expected" % dsn 1636 t, v = sys.exc_info()[:2] 1637 print ' ', t 1638 print ' ', v 1639 1640 dsn = 'dbname=gnumed_v9 user=any-doc password=abc' 1641 try: 1642 conn = get_connection(dsn=dsn) 1643 except cAuthenticationError: 1644 print "SUCCESS: get_connection(%s) failed as expected" % dsn 1645 t, v = sys.exc_info()[:2] 1646 print ' ', t 1647 print ' ', v 1648 1649 dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc' 1650 conn = get_connection(dsn=dsn, readonly=True) 1651 1652 dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc' 1653 conn = get_connection(dsn=dsn, readonly=False) 1654 1655 dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc' 1656 encoding = 'foo' 1657 try: 1658 conn = get_connection(dsn=dsn, encoding=encoding) 1659 except cEncodingError: 1660 print "SUCCESS: get_connection(%s, %s) failed as expected" % (dsn, encoding) 1661 t, v = sys.exc_info()[:2] 1662 print ' ', t 1663 print ' ', v
1664 #--------------------------------------------------------------------
1665 - def test_exceptions():
1666 print "testing exceptions" 1667 1668 try: 1669 raise cAuthenticationError('no dsn', 'no previous exception') 1670 except cAuthenticationError: 1671 t, v, tb = sys.exc_info() 1672 print t 1673 print v 1674 print tb 1675 1676 try: 1677 raise cEncodingError('no dsn', 'no previous exception') 1678 except cEncodingError: 1679 t, v, tb = sys.exc_info() 1680 print t 1681 print v 1682 print tb
1683 #--------------------------------------------------------------------
1684 - def test_ro_queries():
1685 print "testing run_ro_queries()" 1686 1687 dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc' 1688 conn = get_connection(dsn, readonly=True) 1689 1690 data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': u'select version()'}], return_data=True, get_col_idx=True, verbose=True) 1691 print data 1692 print idx 1693 data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': u'select 1'}], return_data=True, get_col_idx=True) 1694 print data 1695 print idx 1696 1697 curs = conn.cursor() 1698 1699 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': u'select version()'}], return_data=True, get_col_idx=True, verbose=True) 1700 print data 1701 print idx 1702 1703 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': u'select 1'}], return_data=True, get_col_idx=True, verbose=True) 1704 print data 1705 print idx 1706 1707 try: 1708 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': u'selec 1'}], return_data=True, get_col_idx=True, verbose=True) 1709 print data 1710 print idx 1711 except psycopg2.ProgrammingError: 1712 print 'SUCCESS: run_ro_queries("selec 1") failed as expected' 1713 t, v = sys.exc_info()[:2] 1714 print ' ', t 1715 print ' ', v 1716 1717 curs.close()
1718 #--------------------------------------------------------------------
1719 - def test_request_dsn():
1720 conn = get_connection() 1721 print conn 1722 conn.close()
1723 #--------------------------------------------------------------------
1724 - def test_set_encoding():
1725 print "testing set_default_client_encoding()" 1726 1727 enc = 'foo' 1728 try: 1729 set_default_client_encoding(enc) 1730 print "SUCCESS: encoding [%s] worked" % enc 1731 except ValueError: 1732 print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc 1733 t, v = sys.exc_info()[:2] 1734 print ' ', t 1735 print ' ', v 1736 1737 enc = '' 1738 try: 1739 set_default_client_encoding(enc) 1740 print "SUCCESS: encoding [%s] worked" % enc 1741 except ValueError: 1742 print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc 1743 t, v = sys.exc_info()[:2] 1744 print ' ', t 1745 print ' ', v 1746 1747 enc = 'latin1' 1748 try: 1749 set_default_client_encoding(enc) 1750 print "SUCCESS: encoding [%s] worked" % enc 1751 except ValueError: 1752 print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc 1753 t, v = sys.exc_info()[:2] 1754 print ' ', t 1755 print ' ', v 1756 1757 enc = 'utf8' 1758 try: 1759 set_default_client_encoding(enc) 1760 print "SUCCESS: encoding [%s] worked" % enc 1761 except ValueError: 1762 print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc 1763 t, v = sys.exc_info()[:2] 1764 print ' ', t 1765 print ' ', v 1766 1767 enc = 'unicode' 1768 try: 1769 set_default_client_encoding(enc) 1770 print "SUCCESS: encoding [%s] worked" % enc 1771 except ValueError: 1772 print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc 1773 t, v = sys.exc_info()[:2] 1774 print ' ', t 1775 print ' ', v 1776 1777 enc = 'UNICODE' 1778 try: 1779 set_default_client_encoding(enc) 1780 print "SUCCESS: encoding [%s] worked" % enc 1781 except ValueError: 1782 print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc 1783 t, v = sys.exc_info()[:2] 1784 print ' ', t 1785 print ' ', v
1786 #--------------------------------------------------------------------
1787 - def test_connection_pool():
1788 dsn = get_default_dsn() 1789 pool = cConnectionPool(minconn=1, maxconn=2, dsn=None, verbose=False) 1790 print pool 1791 print pool.getconn() 1792 print pool.getconn() 1793 print pool.getconn() 1794 print type(pool.getconn())
1795 #--------------------------------------------------------------------
1796 - def test_list_args():
1797 dsn = get_default_dsn() 1798 conn = get_connection(dsn, readonly=True) 1799 curs = conn.cursor() 1800 curs.execute('select * from clin.clin_narrative where narrative = %s', ['a'])
1801 #--------------------------------------------------------------------
1802 - def test_sanitize_pg_regex():
1803 tests = [ 1804 ['(', '\\('] 1805 , ['[', '\\['] 1806 , [')', '\\)'] 1807 ] 1808 for test in tests: 1809 result = sanitize_pg_regex(test[0]) 1810 if result != test[1]: 1811 print 'ERROR: sanitize_pg_regex(%s) returned "%s", expected "%s"' % (test[0], result, test[1])
1812 #--------------------------------------------------------------------
1813 - def test_is_pg_interval():
1814 status = True 1815 tests = [ 1816 [None, True], # None == NULL == succeeds ! 1817 [1, True], 1818 ['1', True], 1819 ['abc', False] 1820 ] 1821 1822 if not is_pg_interval(): 1823 print 'ERROR: is_pg_interval() returned "False", expected "True"' 1824 status = False 1825 1826 for test in tests: 1827 result = is_pg_interval(test[0]) 1828 if result != test[1]: 1829 print 'ERROR: is_pg_interval(%s) returned "%s", expected "%s"' % (test[0], result, test[1]) 1830 status = False 1831 1832 return status
1833 #--------------------------------------------------------------------
1834 - def test_sanity_check_time_skew():
1835 sanity_check_time_skew()
1836 #--------------------------------------------------------------------
1837 - def test_keyword_expansion():
1838 print "keywords, from database:" 1839 print get_text_expansion_keywords() 1840 print "keywords, cached:" 1841 print get_text_expansion_keywords() 1842 print "'$keyword' expands to:" 1843 print expand_keyword(keyword = u'$dvt')
1844 #--------------------------------------------------------------------
1845 - def test_get_foreign_key_details():
1846 for row in get_foreign_keys2column ( 1847 schema = u'dem', 1848 table = u'identity', 1849 column = u'pk' 1850 ): 1851 print '%s.%s references %s.%s.%s' % ( 1852 row['referencing_table'], 1853 row['referencing_column'], 1854 row['referenced_schema'], 1855 row['referenced_table'], 1856 row['referenced_column'] 1857 )
1858 #--------------------------------------------------------------------
1859 - def test_set_user_language():
1860 # (user, language, result, exception type) 1861 tests = [ 1862 # current user 1863 [None, 'de_DE', True], 1864 [None, 'lang_w/o_tx', False], 1865 [None, None, True], 1866 # valid user 1867 ['any-doc', 'de_DE', True], 1868 ['any-doc', 'lang_w/o_tx', False], 1869 ['any-doc', None, True], 1870 # invalid user 1871 ['invalid user', 'de_DE', None], 1872 ['invalid user', 'lang_w/o_tx', False], # lang checking happens before user checking 1873 ['invalid user', None, True] 1874 ] 1875 for test in tests: 1876 try: 1877 result = set_user_language(user = test[0], language = test[1]) 1878 if result != test[2]: 1879 print "test:", test 1880 print "result:", result, "expected:", test[2] 1881 except psycopg2.IntegrityError, e: 1882 if test[2] is None: 1883 continue 1884 print "test:", test 1885 print "expected exception" 1886 print "result:", e
1887 #--------------------------------------------------------------------
1888 - def test_get_schema_revision_history():
1889 for line in get_schema_revision_history(): 1890 print u' - '.join(line)
1891 #-------------------------------------------------------------------- 1892 # run tests 1893 #test_file2bytea() 1894 #test_get_connection() 1895 #test_exceptions() 1896 #test_ro_queries() 1897 #test_request_dsn() 1898 #test_set_encoding() 1899 #test_connection_pool() 1900 #test_list_args() 1901 #test_sanitize_pg_regex() 1902 #test_is_pg_interval() 1903 #test_sanity_check_time_skew() 1904 #test_keyword_expansion() 1905 #test_get_foreign_key_details() 1906 #test_set_user_language() 1907 test_get_schema_revision_history() 1908 1909 # ====================================================================== 1910