# -*- coding: utf-8 -*- ## $HeadURL$ $LastChangedRevision$ ######################################################################## # # Modules # ######################################################################## ## Packages required by this package import subprocess import re import os import sys import getopt import signal import types import uuid import hashlib import socket import time import apsw import shutil import syslog import traceback ######################################################################## # # Characteristics of this module # ######################################################################## # (not necessary in python) ######################################################################## # # Public functions # ######################################################################## ######################################################################## # # Public functions: entry point related # ######################################################################## #DIFFSYNC: main def main(app_main_ref): errstack = [] # Initialise a stack. Other stack related attributes (e.g. dumpall, # verbosity level) are initialised as module-private data at the top # of this file. set_messaging_parameters(errstack, stack=errstack) # This is a one-time loop! It allows us to jump forward on error. while True: # Register the options ADE will handle (and the functions to handle them). rc = register_options(errstack, "Vd:ivhpn", "version,debug:i,verbose,help,paths,simulate", globals(), "_handle_option_%s") if rc != ok: error(errstack, 'ade_err_misc', [ "register_options() call to register ADE options failed" ]) break rc = app_main_ref(errstack) if rc != ok: error(errstack, 'ade_err_misc', "application's entry function failed") break # This break makes it a one-time loop. break # If necessary, display the error stack. if (rc != ok): display_error_stack(errstack) set_messaging_parameters(errstack, stack=errstack) # Convert $rc into a Unix exit code and exit. _exit(errstack, rc) ######################################################################## # # Public functions: error stack related # ######################################################################## #DIFFSYNC: register_error_types def register_error_types(defined_errors): global _registered_defined_errors _registered_defined_errors.update(defined_errors) return ok #DIFFSYNC: display_error_stack def display_error_stack(errstack, displayfunc_ref=None): global _registered_defined_errors # Ideally we'd put 'displayfunc_ref=_error' in the 'def' line # above, but python resolves the references there at *load* time, not at execute # time. _error() is a private function and private functions are # listed *below*, making such a reference here a *forward* reference, which is # not valid in python. With the code below we move the resolution from load # time to run time, at which time a forward reference is fine. if displayfunc_ref is None: displayfunc_ref = _error # For each error frame on the stack ... for i in range(len(errstack)): # Skip all but the top frame if requested so to do if i != 0 and not _dump_whole_error_stack_flag: continue if errstack[i]["err"] not in _registered_defined_errors: internal(errstack, "%s: unknown error" % (errstack[i][err])) error_message = _registered_defined_errors[errstack[i]["err"]]["fmt"] % tuple(errstack[i]["par"]) displayfunc_ref(errstack, {False:"", True:"frame#" + str(i) + ": "}[_dump_whole_error_stack_flag] + error_message) return ok ######################################################################## # # Public functions: string/stream manipulation functions # ######################################################################## #DIFFSYNC: encode_z # (not implemented) #DIFFSYNC: decode_z # (not implemented) #DIFFSYNC: extract_version def extract_version(errstack, svnstring): m = re.search("\$HeadURL: .*?/(trunk)/.*?\$ \$LastChangedRevision: (\d+) \$", svnstring) if m is not None: return ok, "svn/trunk/" + m.group(2) m = re.search("\$HeadURL: .*?/tags/([^/]+)/.*?\$ \$LastChangedRevision: \d+ \$", svnstring) if m is not None: return ok, m.group(1) m = re.search("\$HeadURL: .*?/branches/([^/]+)/.*?\$ \$LastChangedRevision: \d+ \$", svnstring) if m is not None: return ok, "svn/branch/" + m.group(1) internal(errstack, "extract_version: handling of \"%s\" not implemented yet" % (svnstring)) #DIFFSYNC: split_string # (not implemented) #DIFFSYNC: validate_regexp def validate_regexp(errstack, regex): try: re.search(regex, 'some dummy text') except re.error: error(errstack, 'ade_err_invalid', regex, 'regexp') return fail return ok #DIFFSYNC: upper_case # (not implemented) #DIFFSYNC: lower_case # (not implemented) ######################################################################## # # Public functions: messaging functions # ######################################################################## #DIFFSYNC: internal def internal(errstack, message): # The error frames on the stack may be relevant to solving the # problem, so dump them. display_error_stack(errstack) _internal(errstack, message) sys.exit(5) #DIFFSYNC: error def error(errstack, defined_errors_hash_key, *errpars): rc = _validate_error_key(errstack, defined_errors_hash_key) if rc != ok: return rc errstack.append({ "err":defined_errors_hash_key, "par":errpars }) return ok #DIFFSYNC: warning def warning(errstack, defined_errors_hash_key, *errpars): rc = _validate_error_key(errstack, defined_errors_hash_key) if rc != ok: return rc errstack.append({ "err":defined_errors_hash_key, "par":errpars }) display_error_stack(errstack, _warning) set_messaging_parameters(errstack, stack=errstack) return ok #DIFFSYNC: info def info(errstack, message): return _info(errstack, message) #DIFFSYNC: debug def debug(errstack, level, message): return _debug(errstack, level, message) #DIFFSYNC: show_help def show_help(errstack): global _usage_callback # Call callbacks to get text into usage_short_text and usage_long_text. (rc, usage_short_text, usage_long_text) = _usage_callback(errstack) if rc != ok: return rc rc, progname = get_progname(errstack) sys.stdout.write('Usage: %s [ ] ' % (progname)) if usage_short_text is not None: sys.stdout.write(usage_short_text) sys.stdout.write('\n') sys.stdout.write('\n') sys.stdout.write('Options: -V | --version display program version\n') sys.stdout.write(' -v | --verbose verbose\n') sys.stdout.write(' -d | --debug= set debug level\n') sys.stdout.write(' -h | --help display this text\n') sys.stdout.write(' -p | --paths list used paths\n') sys.stdout.write(' -n | --simulate simulate (limited effect!)\n') if usage_long_text is not None: sys.stdout.write(usage_long_text) sys.stdout.write('\n') _exit(errstack, 0) #DIFFSYNC: show_bad_usage def show_bad_usage(errstack): rc, progname = get_progname(errstack) sys.stderr.write('%s: ERROR: type \'%s --help\' for correct usage.\n' % (progname, progname)) _exit(errstack, 1) #DIFFSYNC: show_version def show_version(errstack): (rc, version_text) = _version_callback(errstack) if rc != ok: return rc if version_text is not None: sys.stdout.write("%s version %s\n" % (_progname, version_text)) _exit(errstack, 0) #DIFFSYNC: show_paths def show_paths(errstack): unix_rc = 0 (rc, listpaths_text) = _paths_callback(errstack) if rc != ok: display_error_stack(errstack) set_messaging_parameters(errstack, stack=errstack) unix_rc = 1 if listpaths_text is not None: sys.stdout.write(listpaths_text + "\n") _exit(errstack, unix_rc) #DIFFSYNC: ask_question # (not implemented) ######################################################################## # # Public functions: filename manipulation functions # ######################################################################## #DIFFSYNC: get_absolute_path def get_absolute_path(errstack, what, cwd): if cwd is None: cwd = os.getcwd() what = os.path.normpath(what) if what[0] != '/': what = cwd + '/' + what return (ok, what) #DIFFSYNC: check_readable def check_readable(errstack, thing): if not (os.path.isfile(thing) and os.access(thing, os.R_OK)): error(errstack, 'ade_err_misc', '%s: is not a readable file' % (thing)) return fail return ok #DIFFSYNC: check_writable def check_writable(errstack, thing): if (os.path.isfile(thing) and not os.access(thing, os.W_OK)) or not os.access(thing.rsplit('/', 1)[0], os.W_OK): error(errstack, 'ade_err_misc', '%s: is not a writable file' % (thing)) return fail return ok ######################################################################## # # Public functions: file handle manipulation functions # ######################################################################## #DIFFSYNC: get_free_filehandle # (not implemented) #DIFFSYNC: open_string_for_reading # (not implemented) #DIFFSYNC: create_filehandle # (not implemented) #DIFFSYNC: close_filehandle # (not implemented) ######################################################################## # # Public functions: UUID-related # ######################################################################## #DIFFSYNC: create_uuid def create_uuid(errstack): local_uuid = str(uuid.uuid4()) rc = validate_uuid(errstack, local_uuid) if rc != ok: return rc, None return ok, local_uuid #DIFFSYNC: read_uuid def read_uuid(errstack, file): fp = open(file) if fp is None: error(errstack, 'ade_err_misc', '%s: failed to open file for reading' % (file)) return fail local_uuid = ' '.join(fp.readlines()).strip() fp.close() rc = validate_uuid(errstack, local_uuid) if rc != ok: return rc, None return ok, local_uuid #DIFFSYNC: write_uuid def write_uuid(errstack, file, local_uuid): rc = validate_uuid(errstack, local_uuid) if rc != ok: return rc, None fp = open(file, 'w') if fp is None: error(errstack, 'ade_err_misc', '%s: failed to open file for writing' % (file)) return fail fp.write(local_uuid + '\n') fp.close() return ok #DIFFSYNC: validate_uuid def validate_uuid(errstack, uuid): if not re.search(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', uuid): error(errstack, 'ade_err_misc', '%s: invalid uuid' % (uuid)) return fail return ok ######################################################################## # # Public functions: file content manipulation functions # ######################################################################## #DIFFSYNC: open_compressed_file_for_reading # (not implemented) #DIFFSYNC: open_compressed_file_for_writing # (not implemented) #DIFFSYNC: get_md5sum def get_md5sum(errstack, file): f = open(file, 'rb') bytes = f.read() readable_hash = hashlib.md5(bytes).hexdigest() f.close() return ok, readable_hash ######################################################################## # # Public functions: option processing # ######################################################################## #DIFFSYNC: register_options def register_options(errstack, short_opts, long_opts, globals_dict, opt_func_template="option_handler_%s"): global register_opts debug(errstack, 10, "register_options: short_opts=%s, long_opts=%s, opt_func_template=%s" % tuple([ (x,"None")[x is None] for x in [short_opts, long_opts, opt_func_template ] ])) for opt_type in ("short", "long"): if opt_type == "short": opts = short_opts regexp = r'^([a-zA-Z])(:[siof])?(.*)$' else: opts = long_opts regexp = r'^([a-zA-Z][-a-zA-Z0-9]+)(:[siof])?,?(.*)$' if opts is None: continue debug(errstack, 10, "register_options: opt_type=%s, opts=%s, re=%s" % (opt_type, opts, regexp)) #while [ m for m in [ re.search(regexp, opts) ] if m is not None ]: while True: m = re.search(regexp, opts) if not m: break opt = m.group(1) debug(errstack, 10, "register_options: opt=%s" % (opt)) if m.group(2) is None: opt_argc = 0 opt_argt = None else: opt_argc = 1 opt_argt = m.group(2)[1] # Put remainder back in opts for further analysis in the next loop. opts = m.group(3) opt_suff = opt opt_suff = opt_suff.replace("-","_") opt_func = opt_func_template opt_func = opt_func.replace("%s", opt_suff) debug(errstack, 10, "register_options: opt=%s, opt_suff=%s, opt_argc=%s, opt_argt=%s, opt_func=%s, opts=%s" % tuple([ (x,"None")[x is None] for x in [ opt, opt_suff, opt_argc, opt_argt, opt_func, opts ] ])) rc, subref = funcname_to_funcref(errstack, globals_dict, opt_func) if rc != ok: return rc if subref is None: debug(errstack, 10, "register_options: %s: undefined function" % (opt_func)) error(errstack, 'ade_err_undefined', opt_func, "function") return fail _registered_opt[opt] = [ opt_type, subref, opt_argc, opt_argt ] _registered_opts.append(opt) return ok #DIFFSYNC: process_options def process_options(errstack): global _registered_opts, _show_paths_flag, _show_help_flag, _show_version_flag opt_hash = {} debug(errstack, 10, "process_options: assembling hash ...") for opt in _registered_opts: debug(errstack, 10, "process_options: assembling hash entry for %s ..." % (opt)) opt_suff = opt opt_suff = opt_suff.replace("-", "_") debug(errstack, 10, "process_options: dumping _registered_opt[%s]: %s" % (opt_suff, _registered_opt[opt])) opt_type = _registered_opt[opt][0] opt_func = _registered_opt[opt][1] opt_argc = _registered_opt[opt][2] opt_argt = _registered_opt[opt][3] debug(errstack, 10, "process_options: opt=%s, opt_suff=%s, opt_argc=%s, opt_argt=%s, opt_func=%s" % tuple([(x,"None")[x is None] for x in [opt, opt_suff, opt_argc, opt_argt, opt_func]])) getopts_short_opts = "".join([ "%s%s" % (x, ("",":")[_registered_opt[x][2]]) for x in _registered_opt.keys() if _registered_opt[x][0] == "short" ]) getopts_long_opts = [ "%s%s" % (x, ("","=")[_registered_opt[x][2]]) for x in _registered_opt.keys() if _registered_opt[x][0] == "long" ] try: optlist, sys.argv = getopt.getopt(sys.argv[1:], getopts_short_opts, getopts_long_opts) except getopt.GetoptError: show_bad_usage(errstack) for opt_tuple in optlist: if opt_tuple[0].startswith('--'): opt_suff = opt_tuple[0][2:] elif opt_tuple[0].startswith('-'): opt_suff = opt_tuple[0][1:] else: internal(errstack, "process_options: opt_tuple[0] did not start with '--' or '-'") # Call the option handler that was registered for this option. debug(errstack, 10, "process_options: handler is %s" % (_registered_opt[opt_suff][1])) if _registered_opt[opt_suff][2] == 0: _registered_opt[opt_suff][1](errstack) else: _registered_opt[opt_suff][1](errstack, opt_tuple[1]) debug(errstack, 10, "process_options: _show_paths_flag=%s, _show_help_flag=%s, _show_version_flag=%s" % (_show_paths_flag, _show_help_flag, _show_version_flag)) if (_show_paths_flag): rc = show_paths(errstack) internal(errstack, "process_options: show_paths() unexpectedly returned (rc=%d)" % (rc)) if (_show_help_flag): rc = show_help(errstack) internal(errstack, "process_options: show_help() unexpectedly returned (rc=%d)" % (rc)) if (_show_version_flag): rc = show_version(errstack) internal(errstack, "process_options: show_version() unexpectedly returned (rc=%d)" % (rc)) return ok #DIFFSYNC: funcname_to_funcref def funcname_to_funcref(errstack, globals_dict, funcname): #debug(errstack, 10, 'funcname_to_funcref: globals_dict=%s' % globals_dict) #if funcname not in globals_dict: # funcref = None try: globals_dict[funcname] except KeyError: funcref = None else: if type(globals_dict[funcname]) is not types.FunctionType: funcref = None else: funcref = globals_dict[funcname] return ok, funcref ######################################################################## # # Public functions: special functions # ######################################################################## #DIFFSYNC: manage_rolling_status # (not implemented) #DIFFSYNC: manage_cache # (not implemented) #DIFFSYNC: fork_multi # (not implemented) ######################################################################## # # Public functions: user related # ######################################################################## #DIFFSYNC: get_my_group # (not implemented) ######################################################################## # # Public functions: process management functions # ######################################################################## #DIFFSYNC: validate_command # (not implemented) #DIFFSYNC: start_coproc # (not implemented) #DIFFSYNC: evaler def evaler(errstack, command): global _simulate if _simulate: sys.stdout.write('%s\n' % (command)) elif subprocess.run(command, shell=True).returncode != 0: error(errstack, 'ade_err_misc', '%s: command failed (hint: see messages above?)' % (command.split()[0])) return fail return ok ######################################################################## # # Public functions: temporary file related functions # ######################################################################## #DIFFSYNC: register_exit_function # (not implemented) #DIFFSYNC: deregister_exit_function # (not implemented) #DIFFSYNC: register_temp_file def register_temp_file(errstack, item): (rc, abs_item) = get_absolute_path(errstack, item, None) if rc != ok: return rc _registers['delonexit'].append(item) return ok #DIFFSYNC: deregister_temp_file def deregister_temp_file(errstack, item): (rc, abs_item) = get_absolute_path(errstack, item, None) if rc != ok: return rc return _pop_by_value(errstack, 'delonexit', abs_item) #DIFFSYNC: _deregister_all_exit_functions def _deregister_all_exit_functions(errstack): global _registers _registers['callonexit'] = [] return ok #DIFFSYNC: _deregister_all_temp_files def _deregister_all_temp_files(errstack): global _registers _registers['delonexit'] = [] return ok #DIFFSYNC: _delete_all_temp_files def _delete_all_temp_files(errstack): global _registers os.chdir('/') subprocess.call(["rm", "-fr"] + _registers['delonexit']) return _deregister_all_temp_files(errstack) #DIFFSYNC: _call_all_exit_functions def _call_all_exit_functions(errstack): global _registers for item in _registers['callonexit']: item() return _deregister_all_exit_functions(errstack) ######################################################################## # # Public functions: database related functions # ######################################################################## #DIFFSYNC: write_sql # (not implemented) #DIFFSYNC: execute_sql def execute_sql(errstack, dbh, sql_statement): debug(errstack, 10, 'execute_sql: running "%s" ...' % (sql_statement)) try: dbh.execute(sql_statement) except Exception as e: error(errstack, 'ade_err_misc', e) return fail return ok #DIFFSYNC: select_sql def select_sql(errstack, dbh, sql_statement): debug(errstack, 30, 'select_sql: running "%s" ...' % (sql_statement)) try: select_results = dbh.execute(sql_statement).fetchall() except Exception as e: error(errstack, 'ade_err_misc', e) return fail, None debug(errstack, 30, 'select_sql: after running "%s"' % (sql_statement)) return ok, select_results #DIFFSYNC: select_sql_count def select_sql_count(errstack, dbh, sql_statement): debug(errstack, 30, 'select_sql_count: before calling select_sql()') rc, select_results = select_sql(errstack, dbh, sql_statement) if rc != ok: return rc, None debug(errstack, 30, 'select_sql_count: before unpacking tuple') (count,) = select_results[0] debug(errstack, 30, 'select_sql_count: after unpacking tuple; count=%d' % (count)) return ok, count #DIFFSYNC: execute_sql_qm def execute_sql_qm(errstack, cursor, sql_statement, sql_value, errmsg_reformatter=None): debug(errstack, 10, 'execute_sql_qm: sql_statement=%s, sql_value=%s' % (sql_statement, sql_value)) try: cursor.execute(sql_statement, sql_value) except Exception as e: error(errstack, 'ade_err_misc', str(e) if errmsg_reformatter is None else errmsg_reformatter(str(e))) return fail return ok #DIFFSYNC: select_sql_qm def select_sql_qm(errstack, dbh, sql_statement_qm, sql_statement_tuple): debug(errstack, 10, 'select_sql_qm: sql_statement_qm=%s, sql_statement_tuple=%s' % (sql_statement_qm, sql_statement_tuple)) try: select_results = dbh.execute(sql_statement_qm, sql_statement_tuple).fetchall() except Exception as e: error(errstack, 'ade_err_misc', e) return fail, None return ok, select_results #DIFFSYNC: select_sql_count_qm def select_sql_count_qm(errstack, dbh, sql_statement_qm, sql_statement_tuple): debug(errstack, 10, 'select_sql_count_qm: sql_statement_qm=%s, sql_statement_tuple=%s' % (sql_statement_qm, sql_statement_tuple)) (rc, select_results) = select_sql_qm(errstack, dbh, sql_statement_qm, sql_statement_tuple) if rc != ok: return rc, None (select_count,) = select_results[0] return ok, select_count #DIFFSYNC: begin_sql_transaction def begin_sql_transaction(errstack, cursor): global _inside_sql_transaction_flag assert_outside_sql_transaction(errstack) _inside_sql_transaction_flag = True debug(errstack, 10, 'begin_sql_transaction: locking database ...') sql_statement = ''' BEGIN IMMEDIATE TRANSACTION; ''' return execute_sql(errstack, cursor, sql_statement) #DIFFSYNC: end_sql_transaction def end_sql_transaction(errstack, cursor): global _inside_sql_transaction_flag assert_inside_sql_transaction(errstack) _inside_sql_transaction_flag = False debug(errstack, 10, 'end_sql_transaction: unlocking database ...') sql_statement = ''' END TRANSACTION; ''' return execute_sql(errstack, cursor, sql_statement) #DIFFSYNC: assert_inside_sql_transaction def assert_inside_sql_transaction(errstack): global _inside_sql_transaction_flag if not _inside_sql_transaction_flag: internal(errstack, 'assert_inside_sql_transaction: not inside a transaction!') #DIFFSYNC: assert_outside_sql_transaction def assert_outside_sql_transaction(errstack): global _inside_sql_transaction_flag if _inside_sql_transaction_flag: internal(errstack, 'assert_outside_sql_transaction: not outside a transaction!') #DIFFSYNC: show_sql_transaction def show_sql_transaction(errstack, text): debug(errstack, 10, 'show_sql_transaction: %s: sql_inside_transaction_flag=%s' % (text, _inside_sql_transaction_flag)) return ok #DIFFSYNC: connect_sqlite def connect_sqlite(errstack, db_file, db_init_file): global opt_mode debug(errstack, 10, 'connect_sqlite: checking %s is writable ...' % (db_file)) rc = check_writable(errstack, db_file) if rc != ok: return rc, None, None db_init_needed_flag = not os.path.isfile(db_file) debug(errstack, 10, 'connect_sqlite: db_init_needed_flag=%s' % (db_init_needed_flag)) if db_init_needed_flag: debug(errstack, 10, 'connect_sqlite: initialising database ...') rc, conn, cursor = initialise_sqlite(errstack, db_file, db_init_file) if rc != ok: return rc, conn, cursor else: debug(errstack, 10, 'connect_sqlite: connecting to existing database ...') conn = apsw.Connection(db_file) cursor = conn.cursor() debug(errstack, 10, 'connect_sqlite: enforcing foreign keys ...') cursor.execute('PRAGMA busy_timeout = 10000;') cursor.execute('PRAGMA foreign_keys = 1;') return ok, conn, cursor #DIFFSYNC: disconnect_sqlite # (not implemented) #DIFFSYNC: initialise_sqlite def initialise_sqlite(errstack, db_file, db_init_file): global code_schema_conformancy info(errstack, 'initialising database ...') # slurp SQL from init file. try: f = open(db_init_file, 'r') except: error(errstack, 'ade_err_misc', '%s: failed to open DB init file' % (db_init_file)) return fail, None, None sql = f.read() f.close() # Create database's parent directories if needed. dn = os.path.dirname(db_file) if os.path.islink(dn) or (os.path.exists(dn) and not os.path.isdir(dn)): error(errstack, 'ade_err_misc', '%s: exists already but is not a directory') return fail, None, None elif not os.path.isdir(dn): os.makedirs(os.path.dirname(db_file)) register_temp_file(errstack, db_file) # Connect to database for first time. debug(errstack, 10, 'initialise_sqlite: connecting to database (%s) for first time ...' % (db_file)) try: conn = apsw.Connection(db_file) except: error(errstack, 'ade_err_misc', '%s: failed to connect to DB file (hint: is it writable?)' % (db_file)) return fail, None, None debug(errstack, 10, 'initialise_sqlite: connected; getting cursor ...') cursor = conn.cursor() # Initialise to database with above-slurped SQL. debug(errstack, 10, 'initialise_sqlite: executing above-slurped SQL ...') try: cursor.execute(sql) except: conn.close() os.unlink(db_file) error(errstack, 'ade_err_misc', '%s: failed to load DB init file (hint: is its syntax ok?)' % (db_init_file)) return fail, None, None # If we get this far then we initialised the database and should preserve it. deregister_temp_file(errstack, db_file) return ok, conn, cursor #DIFFSYNC: validate_sql_schema_version def validate_sql_schema_version(errstack, cursor, code_schema_conformancy, progname): rc, db_schema_conformancy = get_sql_schema_version(errstack, cursor) if rc != ok: return rc if db_schema_conformancy < code_schema_conformancy: error(errstack, 'ade_err_misc', 'database schema is too old (do you need to run \'%s --upgrade\'?)' % (progname)) return fail elif db_schema_conformancy > code_schema_conformancy: error(errstack, 'ade_err_misc', 'database schema is too new (do you need to upgrade %s?)' % (progname)) return fail return ok #DIFFSYNC: get_sql_schema_version def get_sql_schema_version(errstack, cursor): sql_statement = 'PRAGMA user_version;' # PRAGMA isn't a SELECT, but apsw.execute() treats it like one, returning # the output in a results-like array of tuples. rc, select_results = select_sql(errstack, cursor, sql_statement) if rc != ok: return rc, None (db_schema_conformancy,) = select_results[0] return ok, db_schema_conformancy #DIFFSYNC: set_sql_schema_version def set_sql_schema_version(errstack, cursor, db_schema_conformancy): cursor.execute('PRAGMA user_version = %d;' % (db_schema_conformancy)) return ok #DIFFSYNC: edit_sqlite def edit_sqlite(errstack, cursor, db_file): # Guts # Dump database to SQL file. debug(errstack, 10, 'edit_sqlite: dumping database ...') tmp_sql_file = '/tmp/%s.%s.sql' % (_progname, os.getpid()) # sqlite3's stdin is a terminal, which will make it display # '-- Loading resources from /home/alexis/.sqliterc'. '-batch' # will prevent that. Note that other calls to sqlite3 in this # script have stdin redirected for various reasons, and so do # not need the '-batch' workaround. sqlite3_cmdline = 'sqlite3 -batch %s .dump > %s' % (db_file, tmp_sql_file) debug(errstack, 10, 'edit_sqlite: sqlite3_cmdline=[%s]' % (sqlite3_cmdline)) register_temp_file(errstack, tmp_sql_file) rc = os.system(sqlite3_cmdline) if rc != 0: # We don't know if tmp_sql_file was created or not, so remove quietly. if os.path.isfile(tmp_sql_file): os.unlink(tmp_sql_file) deregister_temp_file(errstack, tmp_sql_file) error(errstack, 'ade_err_misc', 'failed to dump database') return fail # Note checksum debug(errstack, 10, 'edit_sqlite: checksumming ...') rc, preedit_md5sum = get_md5sum(errstack, tmp_sql_file) if rc != ok: os.unlink(tmp_sql_file) deregister_temp_file(errstack, tmp_sql_file) return rc # Edit. debug(errstack, 10, 'edit_sqlite: editing ...') editor_cmdline = '%s %s' % (os.environ['EDITOR'] if 'EDITOR' in os.environ else 'vi', tmp_sql_file) debug(errstack, 10, 'edit_sqlite: editor_cmdline=[%s]' % (editor_cmdline)) os.system(editor_cmdline) if rc != 0: os.unlink(tmp_sql_file) deregister_temp_file(errstack, tmp_sql_file) error(errstack, 'ade_err_misc', 'failed to edit') return fail # Note new checksum debug(errstack, 10, 'edit_sqlite: checksumming again ...') rc, postedit_md5sum = get_md5sum(errstack, tmp_sql_file) if rc != ok: os.unlink(tmp_sql_file) deregister_temp_file(errstack, tmp_sql_file) return rc debug(errstack, 10, 'edit_sqlite: preedit_md5sum=%s, postedit_md5sum=%s' % (preedit_md5sum, postedit_md5sum)) # If no changes do nothing. if preedit_md5sum == postedit_md5sum: info(errstack, 'no changes detected') os.unlink(tmp_sql_file) deregister_temp_file(errstack, tmp_sql_file) return ok # Check changes are loadable (with foreign keys enforced). info(errstack, 'changes detected; validating before applying ...') sqlite_cmdline = 'sh -c \'sed "s/PRAGMA foreign_keys=OFF;/PRAGMA foreign_keys=ON;/" %s | sqlite3\'' % (tmp_sql_file) debug(errstack, 10, 'edit_sqlite: sqlite_cmdline=[%s]' % (sqlite_cmdline)) rc = os.system(sqlite_cmdline) if rc != 0: error(errstack, 'ade_err_misc', 'validation failed (hint: did you introduce a syntax error?)') os.unlink(tmp_sql_file) deregister_temp_file(errstack, tmp_sql_file) return fail # Delete all tables and reload all from within a transaction to ensure database not emptied. # No, that won't work because vacuuming (which is necessary else things appear to still exist # at least to the point where their existence blocks their creation) is not allowed within # a transaction. # # How about we preserve the database just created as part of the load test and slide that # into place? No, we can't do that because that would require closing the database before # and reopening it after and this function doesn't want to be concerned with that kind # of stuff; it just wants to use the passed cursor to do all database accesses. # # Hmm ... okay, so let's load the new SQL into memory (so if it fails we've not deleted # anything yet) and then delete and load. # # Load SQL ... try: f = open(tmp_sql_file) except: internal(errstack, 'edit_sqlite: %s: failed to open' % (tmp_sql_file)) #sql = f.read() # We will load the dump *inside* a transaction so we need to filter out the transaction # delimiters from the dump. We also don't want the dump tampering with out foreign key # settings. sql = ''.join([l for l in f.readlines() if l not in ['PRAGMA foreign_keys=OFF;\n', 'BEGIN TRANSACTION;\n', 'COMMIT;\n']]) f.close() os.unlink(tmp_sql_file) deregister_temp_file(errstack, tmp_sql_file) # it looks like PRAGMA foreign_keys can't be changed inside a transaction. # I only needed to do in order to ensure that *tables* can be dropped in # any order. But by putting it outside the transaction the scope of the # pragma is obviously increased. So it's just as well that the dump # was already loaded earlier as a test *with* foreign_keys enabled. cursor.execute('PRAGMA foreign_keys = 0;') begin_sql_transaction(errstack, cursor) # ... determine old content ... sql_statement = ''' SELECT name FROM sqlite_master WHERE type = 'index' AND name NOT LIKE 'sqlite_autoindex_%'; ''' rc, select_results = select_sql(errstack, cursor, sql_statement) if rc != ok: return rc indexes = [name for (name,) in select_results] sql_statement = ''' SELECT name FROM sqlite_master WHERE type = 'trigger'; ''' rc, select_results = select_sql(errstack, cursor, sql_statement) if rc != ok: return rc triggers = [name for (name,) in select_results] sql_statement = ''' SELECT name FROM sqlite_master WHERE type = 'view'; ''' rc, select_results = select_sql(errstack, cursor, sql_statement) if rc != ok: return rc views = [name for (name,) in select_results] sql_statement = ''' SELECT name FROM sqlite_master WHERE type = 'table'; ''' rc, select_results = select_sql(errstack, cursor, sql_statement) if rc != ok: return rc tables = [name for (name,) in select_results] # ... drop old content ... for view in views: debug(errstack, 10, 'edit_sqlite: dropping view %s ...' % (view)) sql_statement = 'DROP VIEW %s;' % (view) rc = execute_sql(errstack, cursor, sql_statement) if rc != ok: return rc for index in indexes: debug(errstack, 10, 'edit_sqlite: dropping index %s ...' % (index)) # 'DROP' can't be done with question marks. sql_statement = 'DROP INDEX %s;' % (index) rc = execute_sql(errstack, cursor, sql_statement) if rc != ok: return rc for trigger in triggers: debug(errstack, 10, 'edit_sqlite: dropping trigger %s ...' % (trigger)) sql_statement = 'DROP TRIGGER %s;' % (trigger) rc = execute_sql(errstack, cursor, sql_statement) if rc != ok: return rc #assert_inside_sql_transaction(errstack) # Prevent problems related to the order of table dropping. for table in tables: debug(errstack, 10, 'edit_sqlite: dropping table %s ...' % (table)) sql_statement = 'DROP TABLE %s;' % (table) rc = execute_sql(errstack, cursor, sql_statement) if rc != ok: return rc # ... load new SQL ... cursor.execute(sql) assert_inside_sql_transaction(errstack) # See comment at beginning of transaction regarding why # transaction ended and *then* PRAGMA foreign_keys modified. end_sql_transaction(errstack, cursor) cursor.execute('PRAGMA foreign_keys = 1;') return ok #DIFFSYNC: upgrade_database def upgrade_database(errstack, cursor, db_file, code_schema_conformancy, upgrade_fncs): # Because of how mode_upgrade() calls standard_prologue(), we're not currently # in a transaction, which is perfect because we want to disable foreign keys # (because it just all gets too difficult to do database schema changes otherwise) # and disabling foreign keys can only be done outside transactions (once the # transaction starts then it is ignored). debug(errstack, 10, 'upgrade_database: disabling foreign keys ...') cursor.execute('PRAGMA foreign_keys = 0;') rc = begin_sql_transaction(errstack, cursor) if rc != ok: return rc debug(errstack, 10, 'upgrade_database: get database conformancy ...') rc, db_schema_conformancy = get_sql_schema_version(errstack, cursor) if rc != ok: # Don't end transaction! We want to roll back! return rc debug(errstack, 10, 'upgrade_database: db_schema_conformancy=%d, code_schema_conformancy=%d' % (db_schema_conformancy, code_schema_conformancy)) if db_schema_conformancy == code_schema_conformancy: # Don't end transaction! We want to roll back! error(errstack, 'ade_err_misc', 'no upgrade necessary') return fail # Back up database before starting. db_file_backup = '%s.%d' % (db_file,os.getpid()) info(errstack, 'backing up database to %s ...' % (db_file_backup)) shutil.copy(db_file, db_file_backup) # Loop applying updates. while db_schema_conformancy != code_schema_conformancy: info(errstack, 'upgrading database schema from version %d to version %d ...' % (db_schema_conformancy, db_schema_conformancy+1)) debug(errstack, 10, 'upgrade_database: calling %s() ...' % (upgrade_fncs[db_schema_conformancy].__name__)) rc = upgrade_fncs[db_schema_conformancy](errstack, cursor) if rc != ok: # Don't end transaction! We want to roll back! return rc # Refresh old and new schema conformancy values. old_db_schema_conformancy = db_schema_conformancy rc, db_schema_conformancy = get_sql_schema_version(errstack, cursor) if rc != ok: # Don't end transaction! We want to roll back! return rc # Check the upgrade function bumped the database schema conformancy. if db_schema_conformancy == old_db_schema_conformancy: internal(errstack, 'upgrade_database: %s: failed to change db_schema_conformancy' % (upgrade_fncs[db_schema_conformancy].__name__)) rc = end_sql_transaction(errstack, cursor) if rc != ok: return rc cursor.execute('PRAGMA foreign_keys = 1;') return ok #DIFFSYNC: filter # (not implemented) #DIFFSYNC: blank_sql_null # (not implemented) ######################################################################## # # Public functions: variable related functions # ######################################################################## #DIFFSYNC: inherit # (not implemented) #DIFFSYNC: global # (not implemented) ######################################################################## # # Public functions: locking related functions # ######################################################################## #DIFFSYNC: lock def lock(errstack, lock_file): # Create world-readable temporary lock tmp_lock_file = '%s.%d' % (lock_file, os.getpid()) if os.path.exists(tmp_lock_file): os.unlink(tmp_lock_file) old_umask = os.umask(0o022) register_temp_file(errstack, tmp_lock_file) try: fp = open(tmp_lock_file, 'w') except: # Deregister the temp file as we never created it. deregister_temp_file(errstack, tmp_lock_file) error(errstack, 'ade_err_access', tmp_lock_file, 'create'); return fail os.umask(old_umask) fp.write('%d\n' % os.getpid()) fp.close() # If can easily lock, return try: os.link(tmp_lock_file, lock_file) # We haven't registered LOCK_FILE before now 'cos weren't certain it was ours. We are now. register_temp_file(errstack, lock_file) os.unlink(tmp_lock_file) deregister_temp_file(errstack, tmp_lock_file) return ok except: pass # If lock file not empty and not stale; return fp = open(lock_file) pid = fp.readline().rstrip('\n') fp.close() if re.search('^[1-9][0-9]*$', pid) and os.path.exists('/proc/%s' % (pid)): os.unlink(tmp_lock_file) deregister_temp_file(errstack, tmp_lock_file) error(errstack, 'ade_err_misc', 'lock held by pid %s' % (pid)) return fail # Remove corrupt or stale lock file warning(errstack, 'ade_err_misc', '%s: empty or stale; removing ...' % (lock_file)) os.unlink(lock_file) # Try to lock again (this time we are sure that the lock file will be ours so register now) register_temp_file(errstack, lock_file) try: os.link(tmp_lock_file, lock_file) os.unlink(tmp_lock_file) deregister_temp_file(errstack, tmp_lock_file) return ok except: pass # It failed in a way we don't understand internal(errstack, 'lock: locking failed in way not understood') #DIFFSYNC: unlock def unlock(errstack, lock_file): os.unlink(lock_file) return ok ######################################################################## # # Public functions: directory content management functions # ######################################################################## #DIFFSYNC: move_compressed_file # (not implemented) ######################################################################## # # Public functions: miscellaneous # ######################################################################## # (none - and ADE should keep it that way!) ######################################################################## # # Public functions: access module-private variable # ######################################################################## #DIFFSYNC: get_progname def get_progname(errstack): global _progname return ok, _progname #DIFFSYNC: get_simulate def get_simulate(errstack): global _simulate return ok, _simulate #DIFFSYNC: get_verboselevel def get_verboselevel(errstack): global _verboselevel return ok, _verboselevel #DIFFSYNC: set_progname def set_progname(errstack, progname): global _progname _progname = progname return ok #DIFFSYNC: set_simulate # (not implemented) #DIFFSYNC: set_verboselevel # (not implemented) #DIFFSYNC: set_callbacks def set_callbacks(errstack, usage_text_getter, version_text_getter, listpaths_text_getter): global _usage_callback, _version_callback, _paths_callback _usage_callback = usage_text_getter _version_callback = version_text_getter _paths_callback = listpaths_text_getter return ok #DIFFSYNC: set_messaging_parameters def set_messaging_parameters(errstack_ref, stack=None, dumpall=None, logfile=None, level=None, writers=None): global _dump_whole_error_stack_flag, _display_error_stack_log_file_filename, _verboselevel, _display_callback_refs global _writer_id_to_function if stack is not None: # See http://stackoverflow.com/questions/1400608/how-to-empty-a-list-in-python. del stack[:] if dumpall is not None: _dump_whole_error_stack_flag = dumpall if logfile is not None: _display_error_stack_log_file_filename = logfile if level is not None: _verboselevel = level if writers is not None: _display_callback_refs = [ _writer_id_to_function[x] for x in writers ] return ok ######################################################################## # # Module-private functions # ######################################################################## #DIFFSYNC: _handle_option_V def _handle_option_V(errstack): return _handle_option_version(errstack) #DIFFSYNC: _handle_option_d def _handle_option_d(errstack, verboselevel): return _handle_option_debug(errstack, verboselevel) #DIFFSYNC: _handle_option_v def _handle_option_v(errstack): return _handle_option_verbose(errstack) #DIFFSYNC: _handle_option_h def _handle_option_h(errstack): return _handle_option_help(errstack) #DIFFSYNC: _handle_option_p def _handle_option_p(errstack): return _handle_option_paths(errstack) #DIFFSYNC: _handle_option_n def _handle_option_n(errstack): return _handle_option_simulate(errstack) #DIFFSYNC: _handle_option_version def _handle_option_version(errstack): global _show_version_flag _show_version_flag = True return ok #DIFFSYNC: _handle_option_debug def _handle_option_debug(errstack, verboselevel): global _verboselevel debug(errstack, 10, "_handle_option_debug: setting _verboselevel to %s ..." % (verboselevel)) _verboselevel = int(verboselevel) return ok #DIFFSYNC: _handle_option_verbose def _handle_option_verbose(errstack): global _verboselevel _verboselevel = 3 return ok #DIFFSYNC: _handle_option_help def _handle_option_help(errstack): global _show_help_flag _show_help_flag = True return ok #DIFFSYNC: _handle_option_paths def _handle_option_paths(errstack): global _show_paths_flag _show_paths_flag = True return ok #DIFFSYNC: _handle_option_simulate def _handle_option_simulate(errstack): global _simulate _simulate = 1 return ok #DIFFSYNC: _handle_signal def _handle_signal(signum, frame): errstack = [] set_messaging_parameters(errstack, stack=errstack) info(errstack, "clearing up ...") _exit(errstack, 4) #DIFFSYNC: _validate_error_key def _validate_error_key(errstack, defined_errors_hash_key): global _registered_defined_errors if defined_errors_hash_key not in _registered_defined_errors: internal(errstack, '%s: unknown error' % (defined_errors_hash_key)) return ok #DIFFSYNC: _call_registered_message_writers def _call_registered_message_writers(errstack, template, message, level, syslog_level): global _display_callback_refs template = template.replace('%MESSAGE', message) template = template.replace('%LEVEL', str(level)) for writerfunc_ref in _display_callback_refs: writerfunc_ref(errstack, template, level, syslog_level) return ok #DIFFSYNC: _push # (not implemented) #DIFFSYNC: _pop # (not implemented) #DIFFSYNC: _pop_by_value def _pop_by_value(errstack, listname, item): global _registers _registers[listname] = [x for x in _registers[listname] if x != item] return ok #DIFFSYNC: _pop_all # (not implemented) #DIFFSYNC: _join # (not implemented) #DIFFSYNC: _internal def _internal(errstack, message): return _call_registered_message_writers(errstack, "INTERNAL ERROR: %MESSAGE", message, 1, syslog.LOG_CRIT) #DIFFSYNC: _error def _error(errstack, message): return _call_registered_message_writers(errstack, "ERROR: %MESSAGE", message, 1, syslog.LOG_ERR) #DIFFSYNC: _warning def _warning(errstack, message): return _call_registered_message_writers(errstack, "WARNING: %MESSAGE", message, 2, syslog.LOG_WARNING) #DIFFSYNC: _info def _info(errstack, message): return _call_registered_message_writers(errstack, "INFO: %MESSAGE", message, 3, syslog.LOG_INFO) #DIFFSYNC: _debug def _debug(errstack, level, message): return _call_registered_message_writers(errstack, "DEBUG[%LEVEL]: %MESSAGE", message, level, syslog.LOG_DEBUG) #DIFFSYNC: _display_error_stack_stderr def _display_error_stack_stderr(errstack, text, level, syslog_level): global _verboselevel if level > _verboselevel: return ok try: sys.stderr.write("%s: %s\n" % (_progname, text)) except IOError: pass return ok #DIFFSYNC: _display_error_stack_log_file def _display_error_stack_log_file(errstack, text, level, syslog_level): global _display_error_stack_log_file_filename if _display_error_stack_log_file_filename is None: return ok try: fp = open(_display_error_stack_log_file_filename, 'a') except: # This function can't be allowed to recurse by attempting to send a message! return ok fp.write('%s: %s\n' % (time.strftime("%Y/%m/%dT%H:%M:%S"), text)) fp.close() return ok #DIFFSYNC: _display_error_stack_syslog def _display_error_stack_syslog(errstack, text, level, syslog_level): global _display_error_stack_syslog_facility rc, progname = get_progname(errstack) # Without setting the 'ident' (first parameter to syslog.openlog() then the *current* # sys.argv[0] is used, which, given that a load of stuff has been shuffled off by # the time we get to here, is usually confusing crap. syslog.openlog(progname, logoption=syslog.LOG_PID, facility=_display_error_stack_syslog_facility) syslog.syslog(syslog_level, text) syslog.closelog(); return ok #DIFFSYNC: _display_error_stack_dev_null def _display_error_stack_dev_null(errstack, text, level, syslog_level): return ok #DIFFSYNC: _initialise def _initialise(): # There is no point in passing an error stack to register_error_types(), # because if it failed then it cannot report the error since no error formats # have been successfully register. register_error_types(_defined_errors) # Set up auto-clean-up. signal.signal(signal.SIGHUP, _handle_signal) signal.signal(signal.SIGINT, _handle_signal) signal.signal(signal.SIGTERM, _handle_signal) # Function replacements. #_replace_function([], 'old_name', 'new_name') _replace_function([], 'reset_error_stack', 'set_messaging_parameters') _replace_function([], 'eval', 'evaler') _replace_function([], 'lock_simple', 'lock') _replace_function([], 'unlock_simple', 'unlock') #DIFFSYNC: _validate_function # (not implemented) #DIFFSYNC: _replace_function def _replace_function(errstack, old_fncname, new_fncname): code = ''' def %s(errstack, *other_args): global _replace_function_alerted if 'ADE_REPLACE_FUNCTION_SILENT' in os.environ and os.environ['ADE_REPLACE_FUNCTION_SILENT'] != '': pass elif '%s' in _replace_function_alerted: pass else: warning(errstack, 'ade_err_misc', '%s() has been superceded by %s(), call stack is: %%s' %% (', '.join([ frame[2] for frame in traceback.extract_stack()]))) # Note that _replace_function_alerted is a *list*; in shell and python # it is a hash. (This is because python offers easy is-item-in-list # check whereas shell and python don't, but is-item-in-hash is easy.) _replace_function_alerted.append('%s') return %s(errstack, *other_args) ''' % (old_fncname, old_fncname, old_fncname, new_fncname, old_fncname, new_fncname) exec(code, globals()) return ok #DIFFSYNC: _fork_multi_debug # (not implemented) #DIFFSYNC: _exit def _exit(errstack, rc): _delete_all_temp_files(errstack) _call_all_exit_functions(errstack) sys.exit(0 if rc == ok else 1) ######################################################################## # # Public variables # ######################################################################## ok = 0 fail = 1 ######################################################################## # # Module-private variables with public access functions # ######################################################################## _progname = sys.argv[0].rpartition("/")[2] _simulate = False _verboselevel = 2 _paths_callback = None _usage_callback = None _version_callback = None _display_callback_refs = [ _display_error_stack_stderr ] ######################################################################## # # Other module-private variables # ######################################################################## # Initialise private data _defined_errors = { "ade_err_undefined": { "fmt": "%s: undefined %s" }, "ade_err_access": { "fmt": "%s: can't %s" }, "ade_err_convert": { "fmt": "%s: couldn't convert from %s to %s" }, "ade_err_seeabove": { "fmt": "error detected; see higher frames for more information" }, "ade_err_invalid": { "fmt": "%s: invalid %s" }, 'ade_err_misc': { "fmt": "%s" }, "ade_err_eof": { "fmt": "%s: end of file" }, "ade_err_notimplemented": { "fmt": "%s: not implemented" } } _writer_id_to_function = { 'stderr': _display_error_stack_stderr, 'devnull': _display_error_stack_dev_null, 'syslog': _display_error_stack_syslog, 'logfile': _display_error_stack_log_file, } _dump_whole_error_stack_flag = False _display_error_stack_log_file_filename = None _display_error_stack_syslog_facility = syslog.LOG_LOCAL0 _show_paths_flag = False _show_help_flag = False _show_version_flag = False _inside_sql_transaction_flag = False _registered_opt = {} _registers = { 'delonexit':[], 'callonexit':[] } _registered_defined_errors = {} _registered_opts = [] _replace_function_alerted = [] ######################################################################## # # Actual code # ######################################################################## _initialise()