#!/usr/bin/python3 -u # -*- coding: utf-8 -*- # Note '-u' above, which ensures rdw2's output and the output of its child # processes (e.g. labellers) comes out in the right order under cron. app_svnid = '$HeadURL$ $LastChangedRevision$' import sys import subprocess exec(subprocess.Popen(['ade-config', '--format=python'], stdout=subprocess.PIPE, universal_newlines=True).communicate()[0]) sys.path.append(ade_share_prefix + '/include') exec(subprocess.Popen(['rdw2-config', '--format=python'], stdout=subprocess.PIPE, universal_newlines=True).communicate()[0]) import ade import os import apsw import pathlib import re import time import socket import platform import shutil import selectors rdw2_defined_errors = { 'rdw2_err_misc':{'fmt':'%s'}, } # Instantiate option variables (for symmetry only; not needed in Python) opt_mode = None opt_db_file = None opt_force_flag = None opt_update_cron_flag = None # Other globals fmtstrs = { 'cmd': '%-40.40s', 'set_id': '%-14.14s', 'dle_id': '%-25.25s', 'enabled':'%-7.7s', 'schedule_id': '%-10.10s', 'lock_pid': '%-5.5s', 'backup_schedule': '%-18.18s', 'check_schedule': '%-18.18s', 'client_id': '%-12.12s', 'server_id': '%-12.12s', 'medium_id': '%-14.14s', 'client_type_id': '%-7.7s', 'server_type_id': '%-7.7s', 'medium_type_id': '%-13.13s', 'dle_type_id': '%-10.10s', 'client_hostname': '%-24.24s', 'server_hostname': '%-20.20s', 'client_label': '%-36.36s', 'server_label': '%-36.36s', 'medium_label': '%-36.36s', 'dle_label': '%-36.36s', 'client_labelling_method_id': '%-25.25s', 'server_labelling_method_id': '%-25.25s', 'medium_labelling_method_id': '%-25.25s', 'dle_labelling_method_id': '%-25.25s', 'accesspoint_inspector_method_id': '%-40.40s', 'backup_method_id': '%-25.25s', 'dle_accesspoint': '%-25.25s', 'medium_accesspoint': '%-28.28s', 'backup_timestamp': '%-19.19s', 'start_timestamp': '%-19.19s', 'end_timestamp': '%-19.19s', 'backup_duration': '%7.7s', 'medium_last_used_timestamp': '%-19.19s', 'dle_last_backed_up_timestamp': '%-19.19s', 'db_access_mode': '%-10.10s', 'backup_filter': '%-80.80s', 'seconds_since_1970': '%-10.10s', 'suppress': '%-0.0s' } headers = { 'cmd': 'command', 'set_id': 'set', 'dle_id': 'dle', 'enabled':'enabled', 'schedule_id': 'schedule', 'backup_schedule': 'backup', 'check_schedule': 'check', 'lock_pid': 'pid', 'client_id': 'client', 'server_id': 'server', 'medium_id': 'medium', 'client_type_id': 'type', 'server_type_id': 'type', 'medium_type_id': 'type', 'dle_type_id': 'type', 'client_hostname': 'hostname', 'server_hostname': 'hostname', 'client_label': 'label', 'server_label': 'label', 'medium_label': 'label', 'dle_label': 'label', 'client_labelling_method_id': 'lab. method', 'server_labelling_method_id': 'lab. method', 'medium_labelling_method_id': 'lab. method', 'dle_labelling_method_id': 'lab. method', 'accesspoint_inspector_method_id': 'inspector method', 'backup_method_id': 'backup method', 'dle_accesspoint': 'accesspoint', 'medium_accesspoint': 'accesspoint', 'backup_timestamp': 'backup-time', 'start_timestamp': 'started', 'end_timestamp': 'ended', 'backup_duration': 'time', 'medium_last_used_timestamp': 'last used', 'db_access_mode': 'mode', 'dle_last_backed_up_timestamp': 'last backed up', 'backup_filter': 'filter', 'seconds_since_1970': 'timestamp', } inside_transaction_flag = False medium_accesspoints_cached_flag = False lots_of_hyphens = '-' * 200 lots_of_equals = '=' * 200 code_schema_conformancy = 6 work_dir = '/tmp' rdw2_progname = None def rdw2(errstack): global opt_mode, opt_db_file, opt_force_flag, rdw2_progname, opt_update_cron_flag # Register application-specific errors ade.register_error_types(rdw2_defined_errors) # Config loader and re-executor rc = load_config(errstack) if rc != ade.ok: return rc rc = reexecutor(errstack) if rc != ade.ok: return rc # Get the name of the program. rc, rdw2_progname = ade.get_progname(errstack) if rc != ade.ok: return rc # Defaults for options opt_mode = None rc, progname = ade.get_progname(errstack) opt_db_file = os.environ['RDW2_DB_FILE'] if 'RDW2_DB_FILE' in os.environ else '%s/%s.sqlite' % (rdw2_state_prefix, progname) ade.debug(errstack, 10, 'rdw2: opt_db_file=%s' % (opt_db_file)) opt_force_flag = False opt_update_cron_flag = True # Register options ade.debug(errstack, 10, 'rdw2: registering options ...') rc = ade.register_options(errstack, '', 'edit,upgrade,db-file:s,list-sets,add-set,del-set,list-clients,add-client,del-client,list-servers,add-medium,del-medium,list-media,list-dle-media,list-set-media,list-preferred-dle-medium,list-preferred-set-medium,add-dle,del-dle,list-dles,backup-dle,check-dle,backup-set,check-set,list-backups,list-misc,list-medium-accesspoints,list-medium-accesspoint,list-pids,list-all,force,list-filters,add-filter,del-filter,no-update-cron', globals(), 'rdw2_opt_handler_%s') if rc != ade.ok: return rc # Register handler functions ade.debug(errstack, 10, 'rdw2: registering message handlers ...') rc = ade.set_callbacks(errstack, rdw2_usage_help, rdw2_version, rdw2_paths) if rc != ade.ok: return rc # Process options ade.debug(errstack, 10, 'rdw2: processing options ...') rc = ade.process_options(errstack) if rc != ade.ok: return rc # Process arguments if opt_mode is None: ade.show_bad_usage(errstack) # (the rest is done in delegations) # Sanity checks and derivations. db_init_file = '%s/%s-%d.sql' % (rdw2_share_prefix + '/sql', progname, code_schema_conformancy) if not rdw2_server_flag: ade.error(errstack, 'rdw2_err_misc', 'this is not an rdw2 server! (hint: perhaps add \'rdw2_server_flag = True\' to %s/%s?)' % (rdw2_etc_prefix, 'rdw2.py')) return ade.fail # Open DB. rc, conn, cursor = ade.connect_sqlite(errstack, opt_db_file, db_init_file) if rc != ade.ok: return rc # Check database version compliancy. if opt_mode not in ['upgrade', 'edit']: rc = ade.validate_sql_schema_version(errstack, cursor, code_schema_conformancy, rdw2_progname) if rc != ade.ok: conn.close() return rc # Add this server to the database if necessary. ade.debug(errstack, 10, 'rdw2: opt_mode=%s' % (opt_mode)) if opt_mode not in ['edit','upgrade']: rc = validate_or_add_this_server(errstack, cursor) if rc != ade.ok: conn.close() return rc # Delegate ade.debug(errstack, 10, 'rdw2: delegating ...') if opt_mode == 'edit': rc = mode_edit(errstack, cursor, sys.argv, opt_db_file, opt_force_flag) elif opt_mode == 'upgrade': rc = mode_upgrade(errstack, cursor, sys.argv) elif opt_mode == 'list-sets': rc = mode_list_sets(errstack, cursor, sys.argv) elif opt_mode == 'add-set': rc = mode_add_set(errstack, cursor, sys.argv) elif opt_mode == 'del-set': rc = mode_del_set(errstack, cursor, sys.argv) elif opt_mode == 'list-clients': rc = mode_list_clients(errstack, cursor, sys.argv) elif opt_mode == 'add-client': rc = mode_add_client(errstack, cursor, sys.argv) elif opt_mode == 'del-client': rc = mode_del_client(errstack, cursor, sys.argv) elif opt_mode == 'list-media': rc = mode_list_media(errstack, cursor, sys.argv) elif opt_mode == 'add-medium': rc = mode_add_medium(errstack, cursor, sys.argv) elif opt_mode == 'del-medium': rc = mode_del_medium(errstack, cursor, sys.argv) elif opt_mode == 'list-dles': rc = mode_list_dles(errstack, cursor, sys.argv) elif opt_mode == 'add-dle': rc = mode_add_dle(errstack, cursor, sys.argv) elif opt_mode == 'del-dle': rc = mode_del_dle(errstack, cursor, sys.argv) elif opt_mode == 'list-servers': rc = mode_list_servers(errstack, cursor, sys.argv) elif opt_mode == 'list-dle-media': rc = mode_list_dle_media(errstack, cursor, sys.argv) elif opt_mode == 'list-set-media': rc = mode_list_set_media(errstack, cursor, sys.argv) elif opt_mode == 'list-preferred-dle-medium': rc = mode_list_preferred_dle_medium(errstack, cursor, sys.argv) elif opt_mode == 'list-preferred-set-medium': rc = mode_list_preferred_set_medium(errstack, cursor, sys.argv) elif opt_mode == 'list-backups': rc = mode_list_backups(errstack, cursor, sys.argv) elif opt_mode == 'list-misc': rc = mode_list_misc(errstack, cursor, sys.argv) elif opt_mode == 'list-medium-accesspoints': rc = mode_list_medium_accesspoints(errstack, cursor, sys.argv) elif opt_mode == 'list-medium-accesspoint': rc = mode_list_medium_accesspoint(errstack, cursor, sys.argv) elif opt_mode == 'backup-dle': rc = mode_backup_dle(errstack, cursor, sys.argv) elif opt_mode == 'check-dle': rc = mode_check_dle(errstack, cursor, sys.argv) elif opt_mode == 'backup-set': rc = mode_backup_set(errstack, cursor, sys.argv) elif opt_mode == 'check-set': rc = mode_check_set(errstack, cursor, sys.argv) elif opt_mode == 'list-all': rc = mode_list_all(errstack, cursor, sys.argv) elif opt_mode == 'list-pids': rc = mode_list_pids(errstack, cursor, sys.argv) elif opt_mode == 'list-filters': rc = mode_list_filters(errstack, cursor, sys.argv) elif opt_mode == 'add-filter': rc = mode_add_filter(errstack, cursor, sys.argv) elif opt_mode == 'del-filter': rc = mode_del_filter(errstack, cursor, sys.argv) else: ade.internal(errstack, 'rdw2: %s: unhandled value for opt_mode' % (opt_mode)) if rc != ade.ok: return rc # Clean up ade.debug(errstack, 10, 'rdw2: closing database ...') conn.close() return ade.ok ######################################################################## # # Frontend functions # ######################################################################## def mode_upgrade(errstack, cursor, argv): global code_schema_conformancy if len(argv) != 0: ade.show_bad_usage(errstack) # Standard prologue tries to purge various stale locks and enter # a pid in the locks table but if we're upgrading then it's possible # that none of that exists. So this needs to be done without any of # that. # Now here's the body. Do the upgrade. rc = ade.upgrade_database(errstack, cursor, opt_db_file, code_schema_conformancy, [ upgrade_from_0_to_1, upgrade_from_1_to_2, upgrade_from_2_to_3, upgrade_from_3_to_4, upgrade_from_4_to_5, upgrade_from_5_to_6 ]) if rc != ade.ok: # Don't bother trying to clean up. #errstack2 = [] #standard_epilogue(errstack2, cursor, True) return rc # Standard epilogue likewise not done. return ade.ok def mode_edit(errstack, cursor, argv, db_file, force_flag): if len(argv) != 0: ade.show_bad_usage(errstack) # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, False, 'exclusive' if not force_flag else 'shared', '%s --edit' % (progname)) if rc != ade.ok: return rc rc = ade.edit_sqlite(errstack, cursor, db_file) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc rc = write_crontab(errstack, cursor) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_sets(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-sets' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_sets(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_add_set(errstack, cursor, argv): if len(argv) != 2: ade.show_bad_usage(errstack) set_id = argv[0] schedule_id = argv[1] ade.debug(errstack, 10, 'mode_add_set: set_id=%s, schedule_id=\'%s\'' % (set_id, schedule_id)) # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --add-set' % (progname)) if rc != ade.ok: return rc rc = add_set(errstack, cursor, set_id, schedule_id) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc rc = write_crontab(errstack, cursor) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Standard epilogue rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_del_set(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) set_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --del-set' % (progname)) if rc != ade.ok: return rc rc = del_set(errstack, cursor, set_id) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc rc = write_crontab(errstack, cursor) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Standard epilogue rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_list_clients(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-clients' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_clients(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_add_client(errstack, cursor, argv): if len(argv) != 4: ade.show_bad_usage(errstack) client_id = argv[0] client_type_id = argv[1] client_hostname = argv[2] client_labelling_method_id = argv[3] ade.debug(errstack, 10, 'mode_add_client: client_id=%s, client_type_id=%s, client_hostname=%s, client_labelling_method_id=%s' % (client_id, client_type_id, client_hostname, client_labelling_method_id)) # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --add-client' % (progname)) if rc != ade.ok: return rc rc = add_client(errstack, cursor, client_id, client_type_id, client_hostname, client_labelling_method_id) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Standard epilogue rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_del_client(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) client_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --del-client' % (progname)) if rc != ade.ok: return rc rc = del_client(errstack, cursor, client_id) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Standard epilogue rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_list_media(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-media' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_media(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_add_medium(errstack, cursor, argv): if len(argv) != 4: ade.show_bad_usage(errstack) medium_id = argv[0] medium_type_id = argv[1] medium_labelling_method_id = argv[2] medium_accesspoint = argv[3] ade.debug(errstack, 10, 'mode_add_medium: medium_id=%s, medium_type_id=%s, medium_labelling_method_id=%s, medium_accesspoint=%s' % (medium_id, medium_type_id, medium_labelling_method_id, medium_accesspoint)) # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --add-medium' % (progname)) if rc != ade.ok: return rc rc = add_medium(errstack, cursor, medium_id, medium_type_id, medium_labelling_method_id, medium_accesspoint) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Standard epilogue rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_del_medium(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) medium_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --del-medium' % (progname)) if rc != ade.ok: return rc rc = del_medium(errstack, cursor, medium_id) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Standard epilogue rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_list_dles(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-dles' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_dles(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_add_dle(errstack, cursor, argv): if len(argv) != 7: ade.show_bad_usage(errstack) dle_id = argv[0] set_id = argv[1] dle_type_id = argv[2] client_id = argv[3] dle_accesspoint = argv[4] backup_method_id = argv[5] dle_labelling_method_id = argv[6] ade.debug(errstack, 10, 'mode_add_dle: dle_id=%s, set_id=%s, dle_type_id=%s, client_id=%s, dle_accesspoint=%s, backup_method_id=%s' % (dle_id, set_id, dle_type_id, client_id, dle_accesspoint, backup_method_id)) # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --add-dle' % (progname)) if rc != ade.ok: return rc rc = add_dle(errstack, cursor, dle_id, set_id, dle_type_id, client_id, dle_accesspoint, backup_method_id, dle_labelling_method_id) if rc != ade.ok: return rc # Standard epilogue rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc return ade.ok def mode_del_dle(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) dle_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --del-dle' % (progname)) if rc != ade.ok: return rc rc = del_dle(errstack, cursor, dle_id) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Standard *fast* epilogue rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_list_servers(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-servers' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_servers(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_dle_media(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 1: ade.show_bad_usage(errstack) dle_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-dle-media' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_dle_media(errstack, cursor, dle_id) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_preferred_dle_medium(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 1: ade.show_bad_usage(errstack) dle_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-preferred-dle-medium' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_preferred_dle_medium(errstack, cursor, dle_id) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_set_media(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 1: ade.show_bad_usage(errstack) set_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-set-media' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_set_media(errstack, cursor, set_id) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_preferred_set_medium(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 1: ade.show_bad_usage(errstack) set_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-preferred-set-medium' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_preferred_set_medium(errstack, cursor, set_id) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_medium_accesspoints(errstack, cursor, argv): if len(argv) != 0: ade.show_bad_usage(errstack) # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-medium-accesspoints' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_medium_accesspoints(errstack, cursor) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_medium_accesspoint(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) medium_id = argv[0] # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-medium-accesspoint' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_medium_accesspoint(errstack, cursor, medium_id) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_backups(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-backups' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_backups(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_misc(errstack, cursor, argv): if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-misc' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_methods(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_types(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_schedules(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_pids(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue - 'peeklocks' mode can *only* be used to examine the 'locks' table!!! rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'peeklocks', '%s --list-pids' % (progname)) if rc != ade.ok: return rc rc = list_pids(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_check_dle(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) dle_id = argv[0] ade.debug(errstack, 10, 'mode_check_dle: dle_id=%s' % (dle_id)) rc = mode_check_or_backup_dle(errstack, cursor, dle_id, 'check') if rc != ade.ok: return rc return ade.ok def mode_backup_dle(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) dle_id = argv[0] ade.debug(errstack, 10, 'mode_backup_dle: dle_id=%s' % (dle_id)) rc = mode_check_or_backup_dle(errstack, cursor, dle_id, 'backup') if rc != ade.ok: return rc return ade.ok def mode_check_or_backup_dle(errstack, cursor, dle_id, backup_or_check): # On error there is no need to roll any of this first bit back because it # is all in a single transaction. rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --%s-dle %s' % (progname, backup_or_check, dle_id)) if rc != ade.ok: return rc # Row lock our resources rc = lock_dle(errstack, cursor, dle_id) if rc != ade.ok: return rc # Get preferred medium and medium_accesspoint ade.info(errstack, '%s: choosing medium ...' % (dle_id)) rc, medium_id, medium_accesspoint = get_preferred_dle_medium_and_its_accesspoint(errstack, cursor, dle_id) if rc != ade.ok: return rc, None, None rc = lock_medium(errstack, cursor, medium_id) if rc != ade.ok: return rc # Now we've locked resources we can end the transaction. ade.end_sql_transaction(errstack, cursor) rc = check_or_backup_dle(errstack, cursor, dle_id, backup_or_check, medium_id, medium_accesspoint) if rc != ade.ok: ade.debug(errstack, 10, 'mode_check_or_backup_dle: this should appear on dle check or backup error') # It looks like I'm missing a call to unlock(backups) ... but where is the call to lock(backups)? errstack2 = [] ade.begin_sql_transaction(errstack2, cursor) unlock_medium(errstack2, cursor, medium_id) unlock_dle(errstack2, cursor, dle_id) standard_epilogue(errstack2, cursor, False) return rc # Not quite standard epilogue ade.begin_sql_transaction(errstack, cursor) unlock_medium(errstack, cursor, medium_id) unlock_dle(errstack, cursor, dle_id) rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_check_set(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) set_id = argv[0] ade.debug(errstack, 10, 'mode_check_set: set_id=%s' % (set_id)) rc = mode_check_or_backup_set(errstack, cursor, set_id, 'check') if rc != ade.ok: return rc return ade.ok def mode_backup_set(errstack, cursor, argv): if len(argv) != 1: ade.show_bad_usage(errstack) set_id = argv[0] ade.debug(errstack, 10, 'mode_backup_set: set_id=%s' % (set_id)) rc = mode_check_or_backup_set(errstack, cursor, set_id, 'backup') if rc != ade.ok: return rc return ade.ok def mode_check_or_backup_set(errstack, cursor, set_id, backup_or_check): rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --%s-set %s' % (progname, backup_or_check, set_id)) if rc != ade.ok: return rc # Check the set exists otherwise next functions will bawk. (Beware this code is also # in del_set().) sql_statement = ''' SELECT COUNT(*) FROM sets WHERE set_id = ?; ''' sql_value = (set_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: set does not exist' % (set_id)) return ade.fail # Get preferred medium and medium_accesspoint ade.info(errstack, '%s: choosing medium ...' % (set_id)) rc, medium_id, medium_accesspoint = get_preferred_set_medium_and_its_accesspoint(errstack, cursor, set_id) if rc != ade.ok: return rc # Row lock our resources. rc = lock_set(errstack, cursor, set_id) if rc != ade.ok: return rc rc = lock_medium(errstack, cursor, medium_id) if rc != ade.ok: return rc # Now we've locked resources we can end the transaction. ade.end_sql_transaction(errstack, cursor) rc = check_or_backup_set(errstack, cursor, set_id, backup_or_check, medium_id, medium_accesspoint) if rc != ade.ok: errstack2 = [] ade.begin_sql_transaction(errstack2, cursor) unlock_medium(errstack2, cursor, medium_id) unlock_set(errstack2, cursor, set_id) standard_epilogue(errstack2, cursor, False) return rc # Not quite standard epilogue ade.begin_sql_transaction(errstack, cursor) unlock_medium(errstack, cursor, medium_id) unlock_set(errstack, cursor, set_id) rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_list_all(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-all' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_all(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_list_filters(errstack, cursor, argv): global fmtstrs, headers, lots_of_hyhens if len(argv) != 0: regexp = '(?:%s)' % '|'.join(argv) else: regexp = '.*' # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, True, True, 'shared', '%s --list-filters' % (progname)) if rc != ade.ok: return rc # 'rdw2 --list-whatever | less' and quit before end of listing produces SIGPIPE. Trap it. try: rc = list_filters(errstack, cursor, regexp) if rc != ade.ok: errstack2 = [] standard_epilogue(errstack2, cursor, True) return rc except BrokenPipeError: pass # Standard epilogue rc = standard_epilogue(errstack, cursor, True) if rc != ade.ok: return rc return ade.ok def mode_add_filter(errstack, cursor, argv): if len(argv) != 4: ade.show_bad_usage(errstack) backup_filter = argv[0] set_id = argv[1] dle_id = argv[2] backup_method_id = argv[3] ade.debug(errstack, 10, 'mode_add_filter: backup_filter=%s, set_id=%s, dle_id=%s, backup_method_id=%s' % (backup_filter, set_id, dle_id, backup_method_id)) # Standard prologue but don't commit transaction (that's the "False") because # we're going to lock some stuff first, which should be part of the transaction. rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --add-filter' % (progname)) if rc != ade.ok: return rc # Check passed things are not specified or are defined. if set_id != '': rc = check_set_exists(errstack, cursor, set_id) if rc != ade.ok: return rc if dle_id != '': rc = check_dle_exists(errstack, cursor, dle_id) if rc != ade.ok: return rc if backup_method_id != '': rc = check_backup_method_exists(errstack, cursor, backup_method_id) if rc != ade.ok: return rc # Check that the regular expression is a valid regular expression. ade.validate_regexp(errstack, backup_filter) # Row lock our resources (do we really need to do this? I forget the # logic for whether row locking is required). if set_id != '': rc = lock_set(errstack, cursor, set_id) if rc != ade.ok: return rc # Note that we don't lock *both* set and DLE, but only one or the # other (as lock_set() locks all the DLEs in that set). elif dle_id != '': rc = lock_dle(errstack, cursor, dle_id) if rc != ade.ok: return rc # Now we've locked resources we can end the transaction. ade.end_sql_transaction(errstack, cursor) rc = add_filter(errstack, cursor, backup_filter, set_id, dle_id, backup_method_id) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Not quite standard epilogue (because we want to unlock some # stuff as part of the transaction). ade.begin_sql_transaction(errstack, cursor) if set_id != '': unlock_set(errstack, cursor, dle_id) elif dle_id != '': unlock_dle(errstack, cursor, dle_id) rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def mode_del_filter(errstack, cursor, argv): if len(argv) != 4: ade.show_bad_usage(errstack) backup_filter = argv[0] set_id = argv[1] dle_id = argv[2] backup_method_id = argv[3] ade.debug(errstack, 10, 'mode_del_filter: backup_filter=%s, set_id=%s, dle_id=%s, backup_method_id=%s' % (backup_filter, set_id, dle_id, backup_method_id)) # Standard prologue rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, True, 'shared', '%s --add-filter' % (progname)) if rc != ade.ok: return rc # Check passed things are not specified or are defined. if set_id != '': rc = check_set_exists(errstack, cursor, set_id) if rc != ade.ok: return rc if dle_id != '': rc = check_dle_exists(errstack, cursor, dle_id) if rc != ade.ok: return rc if backup_method_id != '': rc = check_backup_method_exists(errstack, cursor, backup_method_id) if rc != ade.ok: return rc # Don't bother validating the regexp. We only do so when we *add* # a filter, not when we delete on. # Row lock our resources (do we really need to do this? I forget the # logic for whether row locking is required). if set_id != '': rc = lock_set(errstack, cursor, set_id) if rc != ade.ok: return rc # Note that we don't lock *both* set and DLE, but only one or the # other (as lock_set() locks all the DLEs in that set). elif dle_id != '': rc = lock_dle(errstack, cursor, dle_id) if rc != ade.ok: return rc # Now we've locked resources we can end the transaction. ade.end_sql_transaction(errstack, cursor) rc = del_filter(errstack, cursor, backup_filter, set_id, dle_id, backup_method_id) if rc != ade.ok: # Don't commit by calling standard_epilogue; we *want* to roll back! return rc # Not quite standard epilogue ade.begin_sql_transaction(errstack, cursor) if set_id != '': unlock_set(errstack, cursor, dle_id) elif dle_id != '': unlock_dle(errstack, cursor, dle_id) rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok ######################################################################## # # Back-end functions # ######################################################################## def add_filter(errstack, cursor, backup_filter, set_id, dle_id, backup_method_id): sql_statement = ''' INSERT INTO backup_filters VALUES (?, ?, ?, ?, ?); ''' sql_value = (backup_filter, set_id if set_id != '' else None, dle_id if dle_id != '' else None, backup_method_id if backup_method_id != '' else None, 'true') #rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value, lambda x : multireplace(x, {'ConstraintError: ':'', '%{MEDIUM_ID}':medium_id, '%{MEDIUM_TYPE_ID}':medium_type_id, '%{MEDIUM_LABELLING_METHOD_ID}':medium_labelling_method_id})) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value, lambda x : multireplace(x, {'ConstraintError: ':'', '%{BACKUP_FILTER}':backup_filter, '%{SET_ID}':set_id, '%{DLE_ID}':dle_id, '%{BACKUP_METHOD_ID}':backup_method_id})) if rc != ade.ok: return rc return ade.ok def del_filter(errstack, cursor, backup_filter, set_id, dle_id, backup_method_id): # Check there is something delete. sql_statement = ''' SELECT COUNT(*) FROM backup_filters WHERE backup_filter = ? AND (set_id IS NULL or set_id = ?) AND (dle_id IS NULL or dle_id = ?) AND (backup_method_id IS NULL or backup_method_id = ?); ''' sql_value = (backup_filter, set_id, dle_id, backup_method_id) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: # Other messages of this 'does not exist' type in this script precede the # message with the name of the thing that does not exist. But for the # backup filters that's more complicated because a filter is defined by # *several* values. Instead, we just say "filter does not exist". # The same applies in the info message next. ade.error(errstack, 'rdw2_err_misc', 'filter does not exist') return ade.fail # Guts ade.info(errstack, 'deleting filter ...') sql_statement = ''' DELETE FROM backup_filters WHERE backup_filter = ? AND set_id %s AND dle_id %s AND backup_method_id %s; ''' % ('= ?' if set_id != '' else 'IS NULL', '= ?' if dle_id != '' else 'IS NULL', '= ?' if backup_method_id != '' else 'IS NULL') sql_value = tuple([x for x in [backup_filter, set_id, dle_id, backup_method_id] if x != '']) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc return ade.ok def list_sets(errstack, cursor, regexp): # Guts sql_statement = ''' SELECT set_id, lock_pid, enabled, schedule_id, (SELECT backup_schedule FROM schedules WHERE schedules.schedule_id = sets.schedule_id), (SELECT check_schedule FROM schedules WHERE schedules.schedule_id = sets.schedule_id) FROM sets ORDER BY set_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) fmtstrs['x'] = ' %s' % (fmtstrs['backup_schedule']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) fmtstrs['y'] = ' %s' % (fmtstrs['check_schedule']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) sys.stdout.write('%s %s %s %s%s%s\n' % (fmtstrs['set_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['schedule_id'], fmtstrs['x'], fmtstrs['y']) % (headers['set_id'], headers['lock_pid'], headers['enabled'], headers['schedule_id'], headers['backup_schedule'], headers['check_schedule'])) sys.stdout.write('%s %s %s %s%s%s\n' % (fmtstrs['set_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['schedule_id'], fmtstrs['x'], fmtstrs['y']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (set_id, lock_pid, enabled, schedule_id, backup_schedule, check_schedule) in select_results: line = '%s %s %s %s%s%s' % (fmtstrs['set_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['schedule_id'], fmtstrs['x'], fmtstrs['y']) % (set_id, lock_pid if lock_pid is not None else '-', enabled, schedule_id, backup_schedule, check_schedule) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) return ade.ok def add_set(errstack, cursor, set_id, schedule_id): # Guts ade.info(errstack, '%s: adding set ...' % (set_id)) # Collect information needed to do the insert. # Then do the insert. sql_statement = ''' INSERT INTO sets VALUES (?, ?, ?, ?); ''' sql_value = (set_id, None, 'true', schedule_id) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value, lambda x : multireplace(x, {'ConstraintError: ':'', '%{SET_ID}':set_id, '%{SCHEDULE_ID}':schedule_id})) if rc != ade.ok: return rc # Then do everything else. return ade.ok def del_set(errstack, cursor, set_id): # Check there is something delete. (Beware this code is also in mode_check_or_backup_set().) sql_statement = ''' SELECT COUNT(*) FROM sets WHERE set_id = ?; ''' sql_value = (set_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: set does not exist' % (set_id)) return ade.fail # Guts ade.info(errstack, '%s: deleting set ...' % (set_id)) # Delete the item from the database. sql_statement = ''' DELETE FROM sets WHERE set_id = ?; ''' sql_value = (set_id,) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # Then do everything else. return ade.ok def list_clients(errstack, cursor, regexp): # Guts sql_statement = ''' SELECT client_id, enabled, client_type_id, client_hostname, client_labelling_method_id, client_label FROM clients ORDER BY client_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) fmtstrs['x'] = ' %s' % (fmtstrs['client_label']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) sys.stdout.write('%s %s %s %s %s%s\n' % (fmtstrs['client_id'], fmtstrs['enabled'], fmtstrs['client_type_id'], fmtstrs['client_hostname'], fmtstrs['client_labelling_method_id'], fmtstrs['x']) % (headers['client_id'], headers['enabled'], headers['client_type_id'], headers['client_hostname'], headers['client_labelling_method_id'], headers['client_label'])) sys.stdout.write('%s %s %s %s %s%s\n' % (fmtstrs['client_id'], fmtstrs['enabled'], fmtstrs['client_type_id'], fmtstrs['client_hostname'], fmtstrs['client_labelling_method_id'], fmtstrs['x']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (client_id, enabled, client_type_id, client_hostname, client_labelling_method_id, client_label) in select_results: line = '%s %s %s %s %s%s' % (fmtstrs['client_id'], fmtstrs['enabled'], fmtstrs['client_type_id'], fmtstrs['client_hostname'], fmtstrs['client_labelling_method_id'], fmtstrs['x']) % (client_id, enabled, client_type_id, client_hostname, client_labelling_method_id, client_label) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) return ade.ok def add_client(errstack, cursor, client_id, client_type_id, client_hostname, client_labelling_method_id): # This is a fast function and therefore allowed to run as a single transaction, which we should still be in! ade.assert_inside_sql_transaction(errstack) # Guts ade.info(errstack, '%s: adding client ...' % (client_id)) # Collect information needed to do the insert. rc, client_label = ade.create_uuid(errstack) if rc != ade.ok: return rc # Then do the insert. sql_statement = ''' INSERT INTO clients VALUES (?, ?, ?, ?, ?, ?); ''' sql_value = (client_id, 'true', client_type_id, client_hostname, client_labelling_method_id, client_label) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value, lambda x : multireplace(x, {'ConstraintError: ':'', '%{CLIENT_ID}':client_id})) if rc != ade.ok: return rc # Then do everything else. sql_statement = ''' SELECT client_labelling_method_cmd FROM client_labelling_methods WHERE client_labelling_method_id = ?; ''' sql_value = (client_labelling_method_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (client_labelling_method_cmd,) = select_results[0] ade.debug(errstack, 10, 'mode_add_client: checking ssh access to %s ...' % (client_hostname)) rc = validate_ssh(errstack, client_hostname) if rc != ade.ok: return rc ade.debug(errstack, 10, 'mode_add_client: labelling %s ...' % (client_hostname)) rc, abs_client_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + client_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_client_labelling_method_cmd, '--debug=%d' % verboselevel, 'add', client_hostname, client_label] ade.debug(errstack, 10, 'mode_add_client: command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'mode_add_client: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'labeller returned non-zero (hint: see error messages above)') return ade.fail return ade.ok def del_client(errstack, cursor, client_id): # Check there is something delete. sql_statement = ''' SELECT COUNT(*) FROM clients WHERE client_id = ?; ''' sql_value = (client_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: client does not exist' % (client_id)) return ade.fail # Guts ade.info(errstack, '%s: deleting client ...' % (client_id)) # Collect information needed to do the delete sql_statement = ''' SELECT client_hostname, client_labelling_method_id, client_label FROM clients WHERE client_id = ?; ''' sql_value = (client_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (client_hostname,client_labelling_method_id,client_label) = select_results[0] sql_statement = ''' SELECT client_labelling_method_cmd FROM client_labelling_methods WHERE client_labelling_method_id = ?; ''' sql_value = (client_labelling_method_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (client_labelling_method_cmd,) = select_results[0] # Delete the item from the database. sql_statement = ''' DELETE FROM clients WHERE client_id = ?; ''' sql_value = (client_id,) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # Then do everything else. ade.debug(errstack, 10, 'mode_del_client: checking ssh access to %s ...' % (client_hostname)) rc = validate_ssh(errstack, client_hostname) if rc != ade.ok: return rc ade.debug(errstack, 10, 'mode_del_client: unlabelling %s ...' % (client_hostname)) rc, abs_client_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + client_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_client_labelling_method_cmd, '--debug=%n' % verboselevel, 'delete', client_hostname, client_label] ade.debug(errstack, 10, 'mode_del_client: unlabelling command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'mode_del_client: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'labeller returned non-zero (hint: see error messages above)') return ade.fail return ade.ok def list_media(errstack, cursor, regexp): # Guts sql_statement = ''' SELECT medium_id, lock_pid, enabled, medium_type_id, medium_labelling_method_id, medium_label, (SELECT datetime(COALESCE(MAX(backup_timestamp),0), 'unixepoch', 'localtime') FROM backups WHERE backups.medium_id = media.medium_id) FROM media ORDER BY medium_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) fmtstrs['x'] = ' %s' % (fmtstrs['medium_label']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) fmtstrs['y'] = ' %s' % (fmtstrs['medium_last_used_timestamp']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) sys.stdout.write('%s %s %s %s %s%s%s\n' % (fmtstrs['medium_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['medium_type_id'], fmtstrs['medium_labelling_method_id'], fmtstrs['x'], fmtstrs['y']) % (headers['medium_id'], headers['lock_pid'], headers['enabled'], headers['medium_type_id'], headers['medium_labelling_method_id'], headers['medium_label'], headers['medium_last_used_timestamp'])) sys.stdout.write('%s %s %s %s %s%s%s\n' % (fmtstrs['medium_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['medium_type_id'], fmtstrs['medium_labelling_method_id'], fmtstrs['x'], fmtstrs['y']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (medium_id, lock_pid, enabled, medium_type_id, medium_labelling_method_id, medium_label, medium_last_used_timestamp) in select_results: line = '%s %s %s %s %s%s%s' % (fmtstrs['medium_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['medium_type_id'], fmtstrs['medium_labelling_method_id'], fmtstrs['x'], fmtstrs['y']) % (medium_id, lock_pid if lock_pid is not None else '-', enabled, medium_type_id, medium_labelling_method_id, medium_label, medium_last_used_timestamp) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) return ade.ok def add_medium(errstack, cursor, medium_id, medium_type_id, medium_labelling_method_id, medium_accesspoint): # Guts ade.info(errstack, '%s: adding medium ...' % (medium_id)) # Collect information needed to do the insert. rc, medium_label = ade.create_uuid(errstack) if rc != ade.ok: return rc # Then do the insert. sql_statement = ''' INSERT INTO media VALUES (?, ?, ?, ?, ?, ?); ''' sql_value = (medium_id, None, 'true', medium_type_id, medium_labelling_method_id, medium_label) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value, lambda x : multireplace(x, {'ConstraintError: ':'', '%{MEDIUM_ID}':medium_id, '%{MEDIUM_TYPE_ID}':medium_type_id, '%{MEDIUM_LABELLING_METHOD_ID}':medium_labelling_method_id})) if rc != ade.ok: return rc # We used to make some grants: # # sql_statement = ''' # INSERT INTO medium_grants VALUES (NULL, ?, NULL, NULL, NULL, NULL); # ''' # sql_value = (medium_id,) # rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) # if rc != ade.ok: # return rc # # but that caused some rather unintuitive effects. So now we just inform the # user that they'll need to set up some grants themselves. rc, progname = ade.get_progname(errstack) ade.warning(errstack, 'rdw2_err_misc', 'grants need to be added to make this medium usable (hint: run \'%s --edit\' and look for the medium_grants table)' % (progname)) # Then do everything else. sql_statement = ''' SELECT medium_labelling_method_cmd FROM medium_labelling_methods WHERE medium_labelling_method_id = ?; ''' sql_value = (medium_labelling_method_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (medium_labelling_method_cmd,) = select_results[0] # Write medium label. rc, abs_medium_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + medium_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_medium_labelling_method_cmd, '--debug=%d' % verboselevel, 'add', medium_accesspoint, medium_label] ade.debug(errstack, 10, 'add_medium: command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'add_medium: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'labeller returned non-zero (hint: see error messages above)') return ade.fail return ade.ok def del_medium(errstack, cursor, medium_id): # Check there is something delete. sql_statement = ''' SELECT COUNT(*) FROM media WHERE medium_id = ?; ''' sql_value = (medium_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: medium does not exist' % (medium_id)) return ade.fail # Guts ade.info(errstack, '%s: deleting medium ...' % (medium_id)) # Collect information needed to do the delete sql_statement = ''' SELECT medium_type_id, medium_labelling_method_id, medium_label FROM media WHERE medium_id = ?; ''' sql_value = (medium_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (medium_type_id, medium_labelling_method_id, medium_label) = select_results[0] ade.debug(errstack, 10, 'del_medium: medium_type_id=%s, medium_labelling_method_id=%s, medium_label=%s' % (medium_type_id, medium_labelling_method_id, medium_label)) sql_statement = ''' SELECT medium_labelling_method_cmd FROM medium_labelling_methods WHERE medium_labelling_method_id = ?; ''' sql_value = (medium_labelling_method_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (medium_labelling_method_cmd,) = select_results[0] ade.debug(errstack, 10, 'del_medium: medium_labelling_method_cmd=%s' % (medium_labelling_method_cmd)) # Find the medium so we can unlabel it. rc = cache_medium_accesspoints(errstack, cursor) if rc != ade.ok: return rc sql_statement = ''' SELECT medium_accesspoint FROM medium_accesspoints, media WHERE medium_accesspoints.medium_label = media.medium_label AND medium_id = ?; ''' sql_value = (medium_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # We have already checked that the medium_id is valid, but we still # need to check that it is accessible. The above select statement # will have returned zero rows if it is now. if len(select_results) == 0: ade.error(errstack, 'rdw2_err_misc', '%s: not accessible (hint: is it mounted/inserted?)' % (medium_id)) return ade.fail (medium_accesspoint,) = select_results[0] rc = uncache_medium_accesspoints(errstack, cursor) if rc != ade.ok: return rc # Delete the item from the database. sql_statement = ''' DELETE FROM backups WHERE medium_id = ?; ''' sql_value = (medium_id,) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc sql_statement = ''' DELETE FROM medium_grants WHERE medium_id = ?; ''' sql_value = (medium_id,) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc sql_statement = ''' DELETE FROM media WHERE medium_id = ?; ''' sql_value = (medium_id,) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # Then do everything else. # Unlabel medium. rc, abs_medium_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + medium_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_medium_labelling_method_cmd, '--debug=%d' % verboselevel, 'delete', medium_accesspoint, medium_label] ade.debug(errstack, 10, 'del_medium: command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'del_medium: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'labeller returned non-zero (hint: see error messages above)') return ade.fail return ade.ok def list_dles(errstack, cursor, regexp): # Guts sql_statement = ''' SELECT dle_id, lock_pid, enabled, set_id, dle_type_id, client_id, dle_accesspoint, backup_method_id, dle_labelling_method_id, dle_label, (select datetime(coalesce(max(backup_timestamp),0), 'unixepoch', 'localtime') from backups where backups.dle_id = dles.dle_id) as dle_last_backed_up_timestamp FROM dles ORDER BY dle_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) fmtstrs['x'] = ' %s' % (fmtstrs['dle_label']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) fmtstrs['y'] = ' %s' % (fmtstrs['dle_last_backed_up_timestamp']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) sys.stdout.write('%s %s %s %s %s %s %s %s %s%s%s\n' % (fmtstrs['dle_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['set_id'], fmtstrs['dle_type_id'], fmtstrs['client_id'], fmtstrs['dle_accesspoint'], fmtstrs['backup_method_id'], fmtstrs['dle_labelling_method_id'], fmtstrs['x'], fmtstrs['y']) % (headers['dle_id'], headers['lock_pid'], headers['enabled'], headers['set_id'], headers['dle_type_id'], headers['client_id'], headers['dle_accesspoint'], headers['backup_method_id'], headers['dle_labelling_method_id'], headers['dle_label'], headers['dle_last_backed_up_timestamp'])) sys.stdout.write('%s %s %s %s %s %s %s %s%s%s%s\n' % (fmtstrs['dle_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['set_id'], fmtstrs['dle_type_id'], fmtstrs['client_id'], fmtstrs['dle_accesspoint'], fmtstrs['backup_method_id'], fmtstrs['dle_labelling_method_id'], fmtstrs['x'], fmtstrs['y']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (dle_id, lock_pid, enabled, set_id, dle_type_id, client_id, dle_accesspoint, backup_method_id, dle_labelling_method_id, dle_label, dle_last_backed_up_timestamp) in select_results: line = '%s %s %s %s %s %s %s %s %s%s%s' % (fmtstrs['dle_id'], fmtstrs['lock_pid'], fmtstrs['enabled'], fmtstrs['set_id'], fmtstrs['dle_type_id'], fmtstrs['client_id'], fmtstrs['dle_accesspoint'], fmtstrs['backup_method_id'], fmtstrs['dle_labelling_method_id'], fmtstrs['x'], fmtstrs['y']) % (dle_id, lock_pid if lock_pid is not None else '-', enabled, set_id, dle_type_id, client_id, dle_accesspoint, backup_method_id, dle_labelling_method_id, dle_label, dle_last_backed_up_timestamp) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) return ade.ok def add_dle(errstack, cursor, dle_id, set_id, dle_type_id, client_id, dle_accesspoint, backup_method_id, dle_labelling_method_id): # Guts start here ade.info(errstack, '%s: adding dle ...' % (dle_id)) # Collect information needed to do the insert. rc, dle_label = ade.create_uuid(errstack) if rc != ade.ok: return rc # Then do the insert. sql_statement = ''' INSERT INTO dles VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); ''' sql_value = (dle_id, None, 'true', set_id, dle_type_id, client_id, dle_accesspoint, backup_method_id, dle_labelling_method_id, dle_label) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value, lambda x : multireplace(x, {'ConstraintError: ':'', '%{DLE_ID}':dle_id, '%{CLIENT_ID}':client_id, '%{DLE_ACCESSPOINT}':dle_accesspoint, '%{BACKUP_METHOD_ID}':backup_method_id, '%{DLE_LABELLING_METHOD_ID}':dle_labelling_method_id})) if rc != ade.ok: return rc # Then do everything else. sql_statement = ''' SELECT dle_labelling_method_cmd FROM dle_labelling_methods WHERE dle_labelling_method_id = ?; ''' sql_value = (dle_labelling_method_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (dle_labelling_method_cmd,) = select_results[0] sql_statement = ''' SELECT client_hostname FROM clients WHERE client_id = ?; ''' sql_value = (client_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (client_hostname,) = select_results[0] ade.debug(errstack, 10, 'add_dle: checking ssh access to %s ...' % (client_hostname)) rc = validate_ssh(errstack, client_hostname) if rc != ade.ok: return rc ade.debug(errstack, 10, 'add_dle: labelling %s ...' % (client_hostname)) rc, abs_dle_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + dle_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_dle_labelling_method_cmd, '--debug=%d' % verboselevel, 'add', client_hostname, dle_accesspoint, dle_label] ade.debug(errstack, 10, 'add_dle: command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'add_dle: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'labeller returned non-zero (hint: see error messages above)') return ade.fail return ade.ok def del_dle(errstack, cursor, dle_id): # Check there is something delete. (Beware this code is also in mode_check_or_backup_dle().) sql_statement = ''' SELECT COUNT(*) FROM dles WHERE dle_id = ?; ''' sql_value = (dle_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: dle does not exist' % (dle_id)) return ade.fail # Guts start here. ade.info(errstack, '%s: deleting dle ...' % (dle_id)) # Collect information needed to do the delete sql_statement = ''' SELECT dle_labelling_method_cmd, dle_label, dle_accesspoint, client_hostname FROM dles, clients, dle_labelling_methods WHERE dles.client_id = clients.client_id AND dles.dle_labelling_method_id = dle_labelling_methods.dle_labelling_method_id AND dle_id = ?; ''' sql_value = (dle_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (dle_labelling_method_cmd, dle_label, dle_accesspoint, client_hostname) = select_results[0] # Delete the item from the database. sql_statement = ''' DELETE FROM backups WHERE dle_id = ?; ''' sql_value = (dle_id,) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc sql_statement = ''' DELETE FROM dles WHERE dle_id = ?; ''' sql_value = (dle_id,) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # Then do everything else. ade.debug(errstack, 10, 'del_dle: checking ssh access to %s ...' % (client_hostname)) rc = validate_ssh(errstack, client_hostname) if rc != ade.ok: return rc ade.debug(errstack, 10, 'del_dle: unlabelling %s ...' % (client_hostname)) rc, abs_dle_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + dle_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_dle_labelling_method_cmd, '--debug=%d' % verboselevel, 'delete', client_hostname, dle_accesspoint, dle_label] ade.debug(errstack, 10, 'del_dle: unlabelling command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'del_dle: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'labeller returned non-zero (hint: see error messages above)') return ade.fail return ade.ok def list_servers(errstack, cursor, regexp): # Guts sql_statement = ''' SELECT server_id, enabled, server_type_id, server_hostname, server_labelling_method_id, server_label FROM servers ORDER BY server_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) fmtstrs['x'] = ' %s' % (fmtstrs['server_label']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) sys.stdout.write('%s %s %s %s %s%s\n' % (fmtstrs['server_id'], fmtstrs['enabled'], fmtstrs['server_type_id'], fmtstrs['server_hostname'], fmtstrs['server_labelling_method_id'], fmtstrs['x']) % (headers['server_id'], headers['enabled'], headers['server_type_id'], headers['server_hostname'], headers['server_labelling_method_id'], headers['server_label'])) sys.stdout.write('%s %s %s %s %s%s\n' % (fmtstrs['server_id'], fmtstrs['enabled'], fmtstrs['server_type_id'], fmtstrs['server_hostname'], fmtstrs['server_labelling_method_id'], fmtstrs['x']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (server_id, enabled, server_type_id, server_hostname, server_labelling_method_id, server_label) in select_results: line = '%s %s %s %s %s%s' % (fmtstrs['server_id'], fmtstrs['enabled'], fmtstrs['server_type_id'], fmtstrs['server_hostname'], fmtstrs['server_labelling_method_id'], fmtstrs['x']) % (server_id, enabled, server_type_id, server_hostname, server_labelling_method_id, server_label) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) return ade.ok def list_dle_media(errstack, cursor, dle_id): # Guts # Map DLE back to set. Why? .... sql_statement = ''' SELECT set_id FROM dles WHERE dle_id = ?;''' sql_value = (dle_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if len(select_results) == 0: ade.error(errstack, 'rdw2_err_misc', '%s: invalid DLE (hint: to see valid DLEs run \'%s --list-dles\')') return ade.fail (set_id,) = select_results[0] rc = list_set_media(errstack, cursor, set_id) if rc != ade.ok: return rc return ade.ok def list_preferred_dle_medium(errstack, cursor, dle_id): # Guts rc, medium_id, medium_accesspoint = get_preferred_dle_medium_and_its_accesspoint(errstack, cursor, dle_id) if rc != ade.ok: return rc sys.stdout.write('%s\n' % (medium_id)) return ade.ok def list_set_media(errstack, cursor, set_id): # Guts sql_statement = ''' SELECT medium_id, medium_last_used_timestamp FROM medium_set_compatibility WHERE set_id = ? AND compatible = 1; ''' sql_value = (set_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc sys.stdout.write('%s %s\n' % (fmtstrs['medium_id'], fmtstrs['medium_last_used_timestamp']) % (headers['medium_id'], headers['medium_last_used_timestamp'])) sys.stdout.write('%s %s\n' % (fmtstrs['medium_id'], fmtstrs['medium_last_used_timestamp']) % (lots_of_hyphens, lots_of_hyphens)) for select_result in select_results: (medium_id, medium_last_used_timestamp) = select_result sys.stdout.write('%s %s\n' % (fmtstrs['medium_id'], fmtstrs['medium_last_used_timestamp']) % (medium_id, medium_last_used_timestamp if medium_last_used_timestamp is not None else '')) return ade.ok def list_preferred_set_medium(errstack, cursor, set_id): # Guts rc, medium_id, medium_accesspoint = get_preferred_set_medium_and_its_accesspoint(errstack, cursor, set_id) if rc != ade.ok: return rc sys.stdout.write('%s\n' % (medium_id)) return ade.ok def list_medium_accesspoints(errstack, cursor): # Guts rc = cache_medium_accesspoints(errstack, cursor) if rc != ade.ok: return rc sql_statement = ''' SELECT medium_type_id, medium_id, medium_accesspoints.medium_label, medium_accesspoint FROM medium_accesspoints LEFT OUTER JOIN media ON medium_accesspoints.medium_label = media.medium_label ORDER BY medium_type_id, medium_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc ade.debug(errstack, 10, 'list_medium_accesspoints: count=%d' % (len(select_results))) sys.stdout.write('%s %s %s %s\n' % (fmtstrs['medium_type_id'], fmtstrs['medium_id'], fmtstrs['medium_accesspoint'], fmtstrs['medium_label']) % (headers['medium_type_id'], headers['medium_id'], headers['medium_accesspoint'], headers['medium_label'])) sys.stdout.write('%s %s %s %s\n' % (fmtstrs['medium_type_id'], fmtstrs['medium_id'], fmtstrs['medium_accesspoint'], fmtstrs['medium_label']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (medium_type_id, medium_id, medium_label, medium_accesspoint) in select_results: sys.stdout.write('%s %s %s %s\n' % (fmtstrs['medium_type_id'], fmtstrs['medium_id'], fmtstrs['medium_accesspoint'], fmtstrs['medium_label']) % (medium_type_id, medium_id, medium_accesspoint, medium_label)) rc = uncache_medium_accesspoints(errstack, cursor) return ade.ok def list_medium_accesspoint(errstack, cursor, medium_id): # Guts rc, medium_accesspoint = get_medium_accesspoint(errstack, cursor, medium_id) if rc != ade.ok: return rc sys.stdout.write('%s\n' % (medium_accesspoint)) return ade.ok def list_backups(errstack, cursor, regexp): # Guts sql_statement = ''' SELECT datetime(backup_timestamp, 'unixepoch', 'localtime') AS backup_timestamp, dle_id, set_id, lock_pid, datetime(start_timestamp, 'unixepoch', 'localtime') AS start_timestamp, datetime(end_timestamp, 'unixepoch', 'localtime') AS end_timestamp, end_timestamp-start_timestamp as backup_duration, medium_id, backup_timestamp AS seconds_since_1970 FROM backups ORDER BY backup_timestamp, dle_id, medium_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) fmtstrs['x'] = ' %s' % (fmtstrs['seconds_since_1970']) if verboselevel >= 3 else '%s' % (fmtstrs['suppress']) sys.stdout.write('%s %s %s %s %s %s %s %s%s\n' % (fmtstrs['backup_timestamp'], fmtstrs['dle_id'], fmtstrs['set_id'], fmtstrs['lock_pid'], fmtstrs['start_timestamp'], fmtstrs['end_timestamp'], fmtstrs['backup_duration'], fmtstrs['medium_id'], fmtstrs['x']) % (headers['backup_timestamp'], headers['dle_id'], headers['set_id'], headers['lock_pid'], headers['start_timestamp'], headers['end_timestamp'], headers['backup_duration'], headers['medium_id'], headers['seconds_since_1970'])) sys.stdout.write('%s %s %s %s %s %s %s %s%s\n' % (fmtstrs['backup_timestamp'], fmtstrs['dle_id'], fmtstrs['set_id'], fmtstrs['lock_pid'], fmtstrs['start_timestamp'], fmtstrs['end_timestamp'], fmtstrs['backup_duration'], fmtstrs['medium_id'], fmtstrs['x']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (backup_timestamp, dle_id, set_id, lock_pid, start_timestamp, end_timestamp, backup_duration, medium_id, seconds_since_1970) in select_results: line = '%s %s %s %s %s %s %s %s%s' % (fmtstrs['backup_timestamp'], fmtstrs['dle_id'], fmtstrs['set_id'], fmtstrs['lock_pid'], fmtstrs['start_timestamp'], fmtstrs['end_timestamp'], fmtstrs['backup_duration'], fmtstrs['medium_id'], fmtstrs['x']) % (backup_timestamp, dle_id, set_id if set_id is not None else '-', lock_pid if lock_pid is not None else '-', start_timestamp if start_timestamp is not None else '-', end_timestamp if end_timestamp is not None else '-', s2ms(backup_duration) if backup_duration is not None else '-', medium_id, seconds_since_1970) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) return ade.ok def list_methods(errstack, cursor, regexp): # Guts sql_statements = [ ''' SELECT 'client labelling methods', group_concat(client_labelling_method_id,', ') FROM client_labelling_methods; ''', ''' SELECT 'dle labelling methods', group_concat(dle_labelling_method_id,', ') FROM dle_labelling_methods; ''', ''' SELECT 'server labelling methods', group_concat(server_labelling_method_id,', ') FROM server_labelling_methods; ''', ''' SELECT 'medium labelling methods', group_concat(medium_labelling_method_id,', ') FROM medium_labelling_methods; ''', ''' SELECT 'backup methods', group_concat(backup_method_id,', ') FROM backup_methods; ''', ''' SELECT 'accesspoint inspector methods', group_concat(accesspoint_inspector_method_id,', ') FROM accesspoint_inspector_methods; ''' ] for sql_statement in sql_statements: rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc for (description, value) in select_results: line = '%-35.35s %s' % (description + ':', value) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) #sys.stdout.write('\n') return ade.ok def list_types(errstack, cursor, regexp): # Guts sql_statements = [ ''' SELECT 'medium types', group_concat(medium_type_id,', ') FROM medium_types; ''', ''' SELECT 'dle types', group_concat(dle_type_id,', ') FROM dle_types; ''', ''' SELECT 'client types', group_concat(client_type_id,', ') FROM client_types; ''', ''' SELECT 'server types', group_concat(server_type_id,', ') FROM server_types; ''' ] for sql_statement in sql_statements: rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc for (description, value) in select_results: line = '%-35.35s %s' % (description + ':', value) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) #sys.stdout.write('\n') return ade.ok def list_schedules(errstack, cursor, regexp): # Guts # sqlite doesn't have LPAD() and RPAD() but it can be simulated. The technique # is to add a *lot* of spaces at the end you want to be padded and then take # the substring starting at the other end. See this article for more details: # https://stackoverflow.com/questions/6576343/how-to-emulate-lpad-rpad-with-sqlite sql_statements = [ ''' SELECT 'backup schedules', group_concat(substr(schedule_id || ':' || backup_schedule || '; ', 1, 25), '') FROM schedules; ''', ''' SELECT 'check schedules', group_concat(substr(schedule_id || ':' || check_schedule || '; ', 1, 25), '') FROM schedules; ''' ] for sql_statement in sql_statements: rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc for (description, value) in select_results: line = '%-35.35s %s' % (description + ':', value) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) #sys.stdout.write('\n') return ade.ok def list_pids(errstack, cursor, regexp): # Guts sql_statement = ''' SELECT lock_pid, db_access_mode, cmd, datetime(start_timestamp, 'unixepoch', 'localtime') AS start_timestamp FROM locks ORDER BY start_timestamp; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc sys.stdout.write('%s %s %s %s\n' % (fmtstrs['lock_pid'], fmtstrs['db_access_mode'], fmtstrs['cmd'], fmtstrs['start_timestamp']) % (headers['lock_pid'], headers['db_access_mode'], headers['cmd'], headers['start_timestamp'])) sys.stdout.write('%s %s %s %s\n' % (fmtstrs['lock_pid'], fmtstrs['db_access_mode'], fmtstrs['cmd'], fmtstrs['start_timestamp']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (lock_pid, db_access_mode, cmd, start_timestamp) in select_results: line = '%s %s %s %s' % (fmtstrs['lock_pid'], fmtstrs['db_access_mode'], fmtstrs['cmd'], fmtstrs['start_timestamp']) % (lock_pid, db_access_mode, cmd, start_timestamp) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) return ade.ok def write_crontab(errstack, cursor): global work_dir, opt_update_cron_flag # Sanity checks and derivations if not opt_update_cron_flag: ade.warning(errstack, 'rdw2_err_misc', 'crontab not updated by request') return ade.ok rc, progname = ade.get_progname(errstack) crontab_tmp = '%s/%s.%s.cron' % (work_dir, progname, os.getpid()) crontab = '/etc/cron.d/%s' % (progname) crontab_bak = rdw2_state_prefix + '/crontab.bak' ade.debug(errstack, 10, 'write_crontab: crontab_tmp=%s, crontab=%s, crontab_bak=%s' % (crontab_tmp, crontab, crontab_bak)) # The crontab and the crontab.bak must be writable or non-existent but in a writable directory. ade.debug(errstack, 10, 'write_crontab: checking crontab and crontab_bak are writable (or dir writable) ...') for filename in [ crontab, crontab_bak ]: if os.path.isfile(filename) and os.access(filename, os.W_OK): continue if os.access(filename.rsplit('/', 1)[0], os.W_OK): continue ade.error(errstack, 'rdw2_err_misc', '%s: is either not writable or not creatable' % (filename)) return ade.fail # Either both crontab and crontab_bak exist (because rdw2 has created them) or neither # exists (because rdw2 --some-cron-changing-option has never been run). Note that them both # existing but being different is checked further down. ade.debug(errstack, 10, 'write_crontab: checking crontab and crontab_bak either both exist or both don\'t exist ...') if os.path.isfile(crontab) != os.path.isfile(crontab_bak): ade.error(errstack, 'rdw2_err_misc', '%s and %s: one exists and not does not (hint: remove the existing or copy it to the non-existent one)' % (crontab, crontab_bak)) return ade.fail # Guts # write crontab entries to temporary crontab ade.debug(errstack, 10, 'write_crontab: writing temporary crontab ...') ade.register_temp_file(errstack, crontab_tmp) try: f = open(crontab_tmp, 'w') except: ade.internal(errstack, 'write_crontab: %s: failed to create temporary crontab' % (crontab_tmp)) sql_statement = ''' SELECT set_id, enabled, (SELECT check_schedule FROM schedules WHERE schedules.schedule_id=sets.schedule_id), (SELECT backup_schedule FROM schedules WHERE schedules.schedule_id=sets.schedule_id) FROM sets ORDER by set_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc disabled_sets = [] rc, verboselevel = ade.get_verboselevel(errstack) for select_result in select_results: (set_id, enabled, check_schedule, backup_schedule) = select_result rdw2_fullpath = '%s/%s' % (rdw2_bin_prefix, progname) check_cmdline = '%s --debug=%d --check-set %s' % (rdw2_fullpath, verboselevel, set_id) backup_cmdline = '%s --debug=%d --backup-set %s' % (rdw2_fullpath, verboselevel, set_id) if enabled == 'true': f.write('%-15s root %s\n' % (check_schedule, check_cmdline)) f.write('%-15s root %s\n' % (backup_schedule, backup_cmdline)) else: f.write('# %-15s root %s # set is disabled\n' % (check_schedule, check_cmdline)) f.write('# %-15s root %s # set is disabled\n' % (backup_schedule, backup_cmdline)) disabled_sets.append(set_id) f.close() # If both files exist and they differ then we won't move the temporary file into place. ade.debug(errstack, 10, 'write_crontab: checking if both exist but differ ...') if os.path.isfile(crontab) and os.path.isfile(crontab_bak): rc, crontab_md5sum = ade.get_md5sum(errstack, crontab) if rc != ade.ok: return rc rc, crontab_bak_md5sum = ade.get_md5sum(errstack, crontab_bak) if rc != ade.ok: return rc if crontab_md5sum != crontab_bak_md5sum: ade.warning(errstack, 'rdw2_err_misc', 'new crontab will not be installed due to intervening manual modificaion (hint: either manually merge %s into %s or remove %s and %s and rerun this command)' % (crontab_tmp, crontab, crontab_bak, crontab)) # Deregister crontab_tmp otherwise it will get deleted by the program's exit # handler and then the text "hint ... manually merge into ..." # can't be acted on. ade.deregister_temp_file(errstack, crontab_tmp) return ade.ok # Put temporary crontab into place. ade.debug(errstack, 10, 'write_crontab: sliding into place ...') os.system('cp %s %s' % (crontab_tmp, crontab)) os.system('cp %s %s' % (crontab_tmp, crontab_bak)) os.unlink(crontab_tmp) ade.deregister_temp_file(errstack, crontab_tmp) ade.debug(errstack, 10, 'write_crontab: checking for disabled sets ...') if len(disabled_sets) > 0: ade.warning(errstack, 'rdw2_err_misc', 'the following sets are disabled: %s' % (', '.join(disabled_sets))) return ade.ok def check_or_backup_dle(errstack, cursor, dle_id, backup_or_check, medium_id, medium_accesspoint): # Guts # Check the dle. rc, backup_method_ok_flag, client_ok_flag = check_dle(errstack, cursor, dle_id) if rc != ade.ok: return rc # Backup the dle. if backup_or_check == 'backup': rc = backup_dle(errstack, cursor, dle_id, None, medium_id, medium_accesspoint, int(time.time())) if rc != ade.ok: return rc return ade.ok def check_dle(errstack, cursor, dle_id): # lock_dle() has already been called and that checked that the dle exists, so here we're just # collecting information, not validing that the dle is defined. ade.debug(errstack, 10, 'check_dle: %s: collecting more information ...' % (dle_id)) sql_statement = ''' SELECT client_id, (SELECT enabled FROM backup_methods WHERE backup_methods.backup_method_id=dles.backup_method_id), (SELECT client_hostname FROM clients WHERE clients.client_id=dles.client_id), dle_accesspoint, (SELECT dle_labelling_method_cmd FROM dle_labelling_methods WHERE dle_labelling_methods.dle_labelling_method_id=dles.dle_labelling_method_id), dle_label FROM dles WHERE dle_id = ?; ''' sql_value = (dle_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: backup_method_ok_flag = True client_ok_flag = True return rc, backup_method_ok_flag, client_ok_flag if len(select_results) != 1: ade.internal(errstack, 'check_dle: %s: %d matching dles were found' % (dle_id, select_count)) # Note that 'enabled' is not the dle's enabled status but the dle's backup method's enabled status! (client_id, enabled, client_hostname, dle_accesspoint, dle_labelling_method_cmd, dle_label) = select_results[0] ade.debug(errstack, 10, 'check_dle: database says client_id=%s, client_hostname=%s, dle_accesspoint=%s, dle_labelling_method_cmd=%s, dle_label=%s' % (client_id, client_hostname, dle_accesspoint, dle_labelling_method_cmd, dle_label)) # Validate the backup method is enabled if enabled != 'true': ade.error(errstack, 'rdw2_err_misc', '%s: dle is meant to be backed up with a method that is disabled!' % (dle_id)) backup_method_ok_flag = False client_ok_flag = True return ade.fail, backup_method_ok_flag, client_ok_flag # Validate client label ade.info(errstack, '%s: verifying dle\'s client\'s label ...' % (dle_id)) rc = validate_client(errstack, cursor, client_id) if rc != ade.ok: backup_method_ok_flag = True client_ok_flag = False return rc, backup_method_ok_flag, client_ok_flag # Verify DLE's label ade.info(errstack, '%s: verifying dle\'s directory\'s label ...' % (dle_id)) ade.debug(errstack, 10, 'check_dle: verifying %s ...' % (dle_id)) rc, abs_dle_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + dle_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_dle_labelling_method_cmd, '--debug=%d' % verboselevel, 'verify', client_hostname, dle_accesspoint, dle_label] ade.debug(errstack, 10, 'check_dle: verifying command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'check_dle: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', '%s: label verifier returned non-zero (hint: see error messages above)' % (dle_id)) backup_method_ok_flag = True client_ok_flag = True return ade.fail, backup_method_ok_flag, client_ok_flag ade.debug(errstack, 10, 'check_dle: verified %s' % (dle_id)) backup_method_ok_flag = True client_ok_flag = True return ade.ok, backup_method_ok_flag, client_ok_flag # set_id passed for logging purposes only def backup_dle(errstack, cursor, dle_id, set_id, medium_id, medium_accesspoint, backup_timestamp): global work_dir # lock_dle() has already been called and that checked that the dle exists, so here we're just # collecting information, not validing that the dle is defined. ade.debug(errstack, 10, 'backup_dle: %s: collecting more information ...' % (dle_id)) sql_statement = ''' SELECT set_id, (SELECT client_hostname FROM clients WHERE clients.client_id=dles.client_id) AS client_hostname, dle_accesspoint, dles.backup_method_id, (SELECT backup_method_cmd FROM backup_methods WHERE backup_methods.backup_method_id=dles.backup_method_id) AS backup_method_cmd, (SELECT increment_support FROM backup_methods WHERE backup_methods.backup_method_id=dles.backup_method_id) AS increment_support, (SELECT dle_labelling_method_cmd FROM dle_labelling_methods WHERE dle_labelling_methods.dle_labelling_method_id=dles.dle_labelling_method_id) AS dle_labelling_method_cmd, dle_label FROM dles WHERE dle_id = ?; ''' sql_value = (dle_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if len(select_results) != 1: ade.internal(errstack, 'backup_dle: %s: %d matching dles were found' % (dle_id, select_count)) (set_id, client_hostname, dle_accesspoint, backup_method_id, backup_method_cmd, increment_support, dle_labelling_method_cmd, dle_label) = select_results[0] ade.debug(errstack, 10, 'backup_dle: set_id=%s, client_hostname=%s, dle_accesspoint=%s, backup_method_id=%s, backup_method_cmd=%s, increment_support=%s, dle_labelling_method_cmd=%s, dle_label=%s' % (set_id, client_hostname, dle_accesspoint, backup_method_id, backup_method_cmd, increment_support, dle_labelling_method_cmd, dle_label)) sql_statement = ''' SELECT backup_filter FROM backup_filters WHERE (set_id IS NULL or set_id = ?) AND (dle_id IS NULL or dle_id = ?) AND (backup_method_id IS NULL OR backup_method_id = ?) AND enabled='true'; ''' sql_value = (set_id, dle_id, backup_method_id) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc backup_filter_list = [] for select_result in select_results: backup_filter_list += [ x for x in select_result[0].splitlines() if not re.search('^[ \t]*(#|$)', x)] ade.debug(errstack, 5, 'backup_dle: backup_filter_list=%s' % (backup_filter_list)) ade.info(errstack, '%s: backing up dle ...' % (dle_id)) rc = ade.begin_sql_transaction(errstack, cursor) if rc != ade.ok: return rc rc = lock_backup(errstack, cursor, backup_timestamp, dle_id, set_id, medium_id) if rc != ade.ok: return rc ade.end_sql_transaction(errstack, cursor) # Do the backup rc, abs_backup_method_cmd = resolve_if_exists(errstack, 'methods/' + backup_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc ade.debug(errstack, 10, 'backup_dle: abs_backup_method_cmd=%s' % (abs_backup_method_cmd)) rc, verboselevel = ade.get_verboselevel(errstack) if increment_support == 'true': rc, progname = ade.get_progname(errstack) list_file = '%s/%s.%s.list-file' % (work_dir, progname, os.getpid()) ade.register_temp_file(errstack, list_file) cmdline = [abs_backup_method_cmd, '--debug=%d' % verboselevel, '--list-file=%s' % (list_file), client_hostname, dle_accesspoint, medium_accesspoint + '/' + set_id + '/' + dle_id, str(backup_timestamp)] else: cmdline = [abs_backup_method_cmd, '--debug=%d' % verboselevel, client_hostname, dle_accesspoint, medium_accesspoint + '/' + set_id + '/' + dle_id, str(backup_timestamp)] ade.debug(errstack, 10, 'backup_dle: backup command is %s' % (cmdline)) try: backup_child = subprocess.Popen(cmdline, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) except FileNotFoundError: ade.internal(errstack, 'backup_dle: %s: failed to execute' % (cmdline[0])) # See https://stackoverflow.com/questions/31833897/python-read-from-subprocess-stdout-and-stderr-separately-while-preserving-order/31867499. # (The first use of the selectors module on that page won't work with Python3, but the second use does. sel = selectors.DefaultSelector() sel.register(backup_child.stdout, selectors.EVENT_READ) sel.register(backup_child.stderr, selectors.EVENT_READ) ok = True while ok: for key, val1 in sel.select(): # Strip trailing newline to make regexp comparison more intuitive. line = key.fileobj.readline().rstrip('\n') if not line: ok = False break # If at least one filter matches then skip the code that displays the message. ade.debug(errstack, 5, 'backup_dle: type(line)=%s, line=[%s]' % (type(line), line)) matched = False for backup_filter in backup_filter_list: ade.debug(errstack, 5, 'backup_dle: type(backup_filter)=%s, backup_filter=[%s]' % (type(backup_filter), backup_filter)) if re.search(backup_filter, line) is not None: matched = True break ade.debug(errstack, 5, 'backup_dle: matched=%s (True means filter out)' % (matched)) if not matched: handle = sys.stdout if key.fileobj is backup_child.stdout else sys.stderr # Display with stripped newline back on again. handle.write(line + '\n') # Just because we called subprocess.Popen() above does not mean the process # has exited and even when we break out of the stdout/stderr-handling loop # directly above, although it means the program has exited, it still doesn't # mean its return code is available. For that we need to call the wait() # method. Then we can inspect the return code. backup_child.wait() # If the backup method wrote a list file then import it back into the database. # This we *do* need to do in a transaction because otherwise it is too slow. if increment_support == 'true': ade.info(errstack, '%s: noting contents of backup ...' % (dle_id)) f = open(list_file, 'r') rel_files = f.readlines() f.close() os.unlink(list_file) ade.deregister_temp_file(errstack, list_file) sql_statement = ''' INSERT INTO backed_up_files VALUES (?, ?, ?); ''' ade.debug(errstack, 10, 'backup_dle: insert #2 ...') rc = ade.begin_sql_transaction(errstack, cursor) if rc != ade.ok: return rc for rel_file in rel_files: ade.debug(errstack, 10, 'backup_dle: inserting (%s, %s, %s) ...' % (backup_timestamp, dle_id, rel_file.strip())) sql_value = (backup_timestamp, dle_id, rel_file.strip()) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc ade.end_sql_transaction(errstack, cursor) # Record backup has just finished in database (see comments above regarding # doing this in a transaction). # # As above, note also that this UPDATE statement is both a record of the end # of the backup *and* a row unlock. rc = ade.begin_sql_transaction(errstack, cursor) if rc != ade.ok: return rc rc = unlock_backup(errstack, cursor, backup_timestamp, dle_id) if rc != ade.ok: return rc ade.end_sql_transaction(errstack, cursor) # At this point the backup has completed. We can now touch the DLE label # as a way to let any client-side checks know that the DLE has been # recently backed up. Unfortunately, we can't reliably interpret the # backup method's exit code to mean that the files actually were backed # up and either touch or not touch the DLE label accordingly. Some backup # methods will only exit non-zero if they really failed. Others may # exit non-zero if even one file was not backed up, perhaps because it # changed during reading. All we can really do is pass the failure up # so that it gets reported, but we can't return ade.fail. ade.debug(errstack, 10, 'backup_dle: touching dle label on %s ...' % (client_hostname)) rc, abs_dle_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + dle_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_dle_labelling_method_cmd, '--debug=%d' % verboselevel, 'touch', client_hostname, dle_accesspoint, dle_label] ade.debug(errstack, 10, 'backup_dle: command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'backup_dle: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'labeller returned non-zero (hint: see error messages above)') return ade.fail # If the backup actually failed then now we can report that. if backup_child.returncode != 0: ade.error(errstack, 'rdw2_err_misc', '%s: backup failed (hint: see error messages above)' % (dle_id)) return ade.fail # And finally, we're really finished. return ade.ok def check_or_backup_set(errstack, cursor, set_id, backup_or_check, medium_id, medium_accesspoint): # Guts rc, passed_dles = check_set(errstack, cursor, set_id) if rc != ade.ok: return rc if backup_or_check == 'backup': rc = backup_set(errstack, cursor, set_id, medium_id, medium_accesspoint, int(time.time()), passed_dles) if rc != ade.ok: return rc return ade.ok def check_set(errstack, cursor, set_id): # lock_set() has already been called and that checked that the set exists, so here we're just # collecting information, not validing that the set is defined. ade.debug(errstack, 10, 'check_set: %s: collecting more information ...' % (set_id)) sql_statement = ''' SELECT COUNT(*) FROM sets WHERE set_id = ?; ''' sql_value = (set_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc, None if select_count != 1: ade.internal(errstack, 'check_set: %s: %d matching sets were found' % (set_id, select_count)) # Check all the dles in the set # The 'enabled' value is true if both the dle *and* the dle's client are both enabled. # client_id and backup_method_id are only needed to help skip calls to check_dle() when # a DLE with the same client id or the same backup method id has already failed *its* checks. sql_statement = ''' SELECT dle_id, CASE WHEN enabled = 'true' AND (SELECT enabled FROM clients WHERE clients.client_id == dles.client_id) = 'true' THEN 'true' ELSE 'false' END AS enabled, client_id, backup_method_id FROM dles WHERE set_id = ? ORDER BY dle_id; ''' sql_value = (set_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc, None if len(select_results) == 0: rc, progname = ade.get_progname(errstack) ade.error(errstack, 'rdw2_err_misc', '%s: empty set (hint: use \'%s --add-dle ...\' to add some dles to this set)' % (set_id, progname)) return ade.fail, None disabled_dles = [] failed_dles = [] passed_dles = [] failed_backup_method_ids = [] failed_client_ids = [] for select_result in select_results: (dle_id,enabled,client_id,backup_method_id) = select_result if enabled == 'false': disabled_dles.append(dle_id) continue if backup_method_id in failed_backup_method_ids: failed_dles.append(dle_id) continue if client_id in failed_client_ids: failed_dles.append(dle_id) continue ade.debug(errstack, 10, 'check_set: calling check_dle(errstack, cursor, %s) ...' % (dle_id)) rc, backup_method_ok_flag, client_ok_flag = check_dle(errstack, cursor, dle_id) if rc != ade.ok: # Display error, reset stack, append this dle to the list of failed dles, move to next dle ade.display_error_stack(errstack) ade.set_messaging_parameters(errstack, stack=errstack) failed_dles.append(dle_id) if not backup_method_ok_flag and backup_method_id not in failed_backup_method_ids: ade.info(errstack, 'the checking of other DLEs using the same backup method will be skipped') failed_backup_method_ids.append(backup_method_id) if not client_ok_flag and client_id not in failed_client_ids: ade.info(errstack, 'the checking of other DLEs on the same client will be skipped') failed_client_ids.append(client_id) continue passed_dles.append(dle_id) if len(disabled_dles) > 0: ade.warning(errstack, 'rdw2_err_misc', 'the following dles or their associated clients are disabled: %s' % (', '.join(disabled_dles))) if len(failed_dles) > 0: ade.warning(errstack, 'rdw2_err_misc', 'the following dles failed their checks: %s' % (', '.join(failed_dles))) return ade.ok, passed_dles def backup_set(errstack, cursor, set_id, medium_id, medium_accesspoint, backup_timestamp, passed_dles): ade.info(errstack, '%s: backing up set ...' % (set_id)) # We don't need to look up the dles in the set because check_set() already did this for us and # it reported the failures and it reported the disableds and it collected the passeds. So # we use that. aggregated_rc = ade.ok for dle_id in passed_dles: # set_id passed for logging purposes only rc = backup_dle(errstack, cursor, dle_id, set_id, medium_id, medium_accesspoint, backup_timestamp) # Don't abort backup up all dles if one fails: display and note the error ... if rc != ade.ok: aggregated_rc = rc ade.display_error_stack(errstack) ade.set_messaging_parameters(errstack, stack=errstack) # .... and return failure only after all dles have been backed up. if aggregated_rc != ade.ok: ade.error(errstack, 'rdw2_err_misc', '%s: the backup of at least one dle in this set failed; see above for details' % (set_id)) return ade.fail, None return ade.ok def list_all(errstack, cursor, regexp): sys.stdout.write('pids\n====\n\n') rc = list_pids(errstack, cursor, regexp) if rc != ade.ok: return rc sys.stdout.write('\n') sys.stdout.write('servers\n=======\n\n') rc = list_servers(errstack, cursor, regexp) if rc != ade.ok: return rc sys.stdout.write('\n') sys.stdout.write('sets\n====\n\n') rc = list_sets(errstack, cursor, regexp) if rc != ade.ok: return rc sys.stdout.write('\n') sys.stdout.write('clients\n=======\n\n') rc = list_clients(errstack, cursor, regexp) if rc != ade.ok: return rc sys.stdout.write('\n') sys.stdout.write('dles\n====\n\n') rc = list_dles(errstack, cursor, regexp) if rc != ade.ok: return rc sys.stdout.write('\n') sys.stdout.write('media\n=====\n\n') rc = list_media(errstack, cursor, regexp) if rc != ade.ok: return rc sys.stdout.write('\n') sys.stdout.write('backups\n=======\n\n') rc = list_backups(errstack, cursor, regexp) if rc != ade.ok: return rc sys.stdout.write('\n') return ade.ok def list_filters(errstack, cursor, regexp): # Guts sql_statement = ''' SELECT backup_filter, set_id, dle_id, backup_method_id, enabled FROM backup_filters ORDER BY set_id, dle_id, backup_method_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc sys.stdout.write('%s %s %s %s %s\n' % (fmtstrs['backup_filter'], fmtstrs['set_id'], fmtstrs['dle_id'], fmtstrs['backup_method_id'], fmtstrs['enabled']) % (headers['backup_filter'], headers['set_id'], headers['dle_id'], headers['backup_method_id'], headers['enabled'])) sys.stdout.write('%s %s %s %s %s\n' % (fmtstrs['backup_filter'], fmtstrs['set_id'], fmtstrs['dle_id'], fmtstrs['backup_method_id'], fmtstrs['enabled']) % (lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens, lots_of_hyphens)) for (backup_filter, set_id, dle_id, backup_method_id, enabled) in select_results: line = '%s %s %s %s %s' % (fmtstrs['backup_filter'], fmtstrs['set_id'], fmtstrs['dle_id'], fmtstrs['backup_method_id'], fmtstrs['enabled']) % (backup_filter, set_id if set_id is not None else '-', dle_id if dle_id is not None else '-', backup_method_id if backup_method_id is not None else '-', enabled) if re.search(regexp, line): sys.stdout.write('%s\n' % (line)) return ade.ok ######################################################################## # # Row lock, process and transaction helpers # ######################################################################## def standard_prologue(errstack, cursor, end_transaction_flag, validate_server_flag, db_access_mode, this_command): # Start transaction. rc = ade.begin_sql_transaction(errstack, cursor) if rc != ade.ok: return rc # If requested, check we're on the right machine. if validate_server_flag: rc = validate_server(errstack, cursor) if rc != ade.ok: return rc # Purge stale locks. rc = purge_stale_set_locks(errstack, cursor) if rc != ade.ok: return rc rc = purge_stale_medium_locks(errstack, cursor) if rc != ade.ok: return rc rc = purge_stale_dle_locks(errstack, cursor) if rc != ade.ok: return rc rc = purge_stale_backup_locks(errstack, cursor) if rc != ade.ok: return rc rc = purge_stale_instance_locks(errstack, cursor) if rc != ade.ok: return rc # Register this process. rc = lock_instance(errstack, cursor, os.getpid(), db_access_mode, this_command) if rc != ade.ok: return rc # If requested, then end the transaction. (The only situation # when this should not be done is when the caller has additional # resources to lock and will themselves end the transaction.) if end_transaction_flag: ade.debug(errstack, 10, 'standard_prologue: calling end_transaction() ...') rc = ade.end_sql_transaction(errstack, cursor) if rc != ade.ok: return rc return ade.ok def standard_epilogue(errstack, cursor, begin_transaction_flag): # If requested, then start the transaction. (The only situation # when this should not be done is when the caller has themselves # already started a transaction in ordet to unlock additional # resources.) if begin_transaction_flag: rc = ade.begin_sql_transaction(errstack, cursor) if rc != ade.ok: return rc # Deregister this process. rc = unlock_instance(errstack, cursor, os.getpid()) if rc != ade.ok: return rc # End transaction. rc = ade.end_sql_transaction(errstack, cursor) if rc != ade.ok: return rc return ade.ok def lock_set(errstack, cursor, set_id): # This function should only be called inside a transaction. ade.assert_inside_sql_transaction(errstack) # Lock database row. rc = sql_lock_row(errstack, cursor, 'sets', 'update', ['set_id'], [set_id], ['lock_pid'], [os.getpid()], 'set', set_id, lambda x : multireplace(x, {'ConstraintError: ':'', '%{SET_ID}':set_id, 'row locked':'set locked'})) if rc != ade.ok: return rc # Lock dles sql_statement = ''' SELECT dle_id FROM dles WHERE dles.set_id = ? AND dles.enabled = 'true' ORDER BY dle_id; ''' sql_value = (set_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc for select_result in select_results: (dle_id,) = select_result rc = lock_dle(errstack, cursor, dle_id) if rc != ade.ok: return rc return ade.ok def unlock_set(errstack, cursor, set_id): # This function should only be called inside a transaction. ade.assert_inside_sql_transaction(errstack) # Unlock dles. sql_statement = ''' SELECT dle_id FROM dles WHERE dles.set_id = ? AND dles.enabled = 'true' ORDER BY dle_id; ''' sql_value = (set_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc for select_result in select_results: (dle_id,) = select_result unlock_dle(errstack, cursor, dle_id) # Unlock database row. rc = sql_unlock_row(errstack, cursor, 'sets', 'update', ['set_id'], [set_id], ['lock_pid'], [None], 'set', set_id, lambda x : multireplace(x, {'ConstraintError: ':'', '%{SET_ID}':set_id})) if rc != ade.ok: return rc return ade.ok def purge_stale_set_locks(errstack, cursor): ade.assert_inside_sql_transaction(errstack) # Get the set id of sets that have a pid ... sql_statement = ''' SELECT set_id, lock_pid FROM sets WHERE lock_pid IS NOT NULL; ''' sql_value = () rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # ... and filter them down the list that are not actually running ... stalely_locked_set_ids = [ set_id for (set_id,lock_pid) in select_results if not os.path.isdir('/proc/%d' % (lock_pid)) ] # ... and unlock those. for stalely_locked_set_id in stalely_locked_set_ids: rc = unlock_set(errstack, cursor, stalely_locked_set_id) if rc != ade.ok: return rc return ade.ok def lock_dle(errstack, cursor, dle_id): # This function should only be called inside a transaction. ade.assert_inside_sql_transaction(errstack) # Lock database row. rc = sql_lock_row(errstack, cursor, 'dles', 'update', ['dle_id'], [dle_id], ['lock_pid'], [os.getpid()], 'dle', dle_id, lambda x : multireplace(x, {'ConstraintError: ':'', '%{DLE_ID}':dle_id, 'row locked':'dle locked'})) if rc != ade.ok: return rc return ade.ok def unlock_dle(errstack, cursor, dle_id): # This function should only be called inside a transaction. ade.assert_inside_sql_transaction(errstack) # Unlock database row. rc = sql_unlock_row(errstack, cursor, 'dles', 'update', ['dle_id'], [dle_id], ['lock_pid'], [None], 'dle', dle_id, lambda x : multireplace(x, {'ConstraintError: ':'', '%{DLE_ID}':dle_id})) if rc != ade.ok: return rc return ade.ok def purge_stale_dle_locks(errstack, cursor): ade.assert_inside_sql_transaction(errstack) sql_statement = ''' SELECT dle_id, lock_pid FROM dles WHERE lock_pid IS NOT NULL; ''' sql_value = () rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc stalely_locked_dle_ids = [ dle_id for (dle_id,lock_pid) in select_results if not os.path.isdir('/proc/%d' % (lock_pid)) ] for stalely_locked_dle_id in stalely_locked_dle_ids: rc = unlock_dle(errstack, cursor, stalely_locked_dle_id) if rc != ade.ok: return rc return ade.ok def lock_medium(errstack, cursor, medium_id): # This function should only be called inside a transaction. ade.assert_inside_sql_transaction(errstack) # Lock database row. rc = sql_lock_row(errstack, cursor, 'media', 'update', ['medium_id'], [medium_id], ['lock_pid'], [os.getpid()], 'medium', medium_id, lambda x : multireplace(x, {'ConstraintError: ':'', '%{MEDIUM_ID}':medium_id, 'row locked':'medium locked'})) if rc != ade.ok: return rc return ade.ok def unlock_medium(errstack, cursor, medium_id): # This function should only be called inside a transaction. ade.assert_inside_sql_transaction(errstack) # Unlock database row. rc = sql_unlock_row(errstack, cursor, 'media', 'update', ['medium_id'], [medium_id], ['lock_pid'], [None], 'medium', medium_id) if rc != ade.ok: return rc return ade.ok def purge_stale_medium_locks(errstack, cursor): ade.assert_inside_sql_transaction(errstack) sql_statement = ''' SELECT medium_id, lock_pid FROM media WHERE lock_pid IS NOT NULL; ''' sql_value = () rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # ... and filter them down the list that are not actually running ... stalely_locked_medium_ids = [ medium_id for (medium_id,lock_pid) in select_results if not os.path.isdir('/proc/%d' % (lock_pid)) ] # ... and unlock those. for stalely_locked_medium_id in stalely_locked_medium_ids: rc = unlock_medium(errstack, cursor, stalely_locked_medium_id) if rc != ade.ok: return rc return ade.ok def lock_backup(errstack, cursor, backup_timestamp, dle_id, set_id, medium_id): ade.assert_inside_sql_transaction(errstack) rc = sql_lock_row(errstack, cursor, 'backups', 'insert', None, None, ['backup_timestamp', 'dle_id', 'set_id', 'lock_pid', 'start_timestamp', 'end_timestamp', 'medium_id'], [backup_timestamp, dle_id, set_id, os.getpid(), int(time.time()), None, medium_id], 'backup', '%d-%s' % (backup_timestamp, dle_id)) if rc != ade.ok: return rc return ade.ok def unlock_backup(errstack, cursor, backup_timestamp, dle_id): rc = sql_unlock_row(errstack, cursor, 'backups', 'update', ['backup_timestamp', 'dle_id'], [backup_timestamp, dle_id], ['end_timestamp','lock_pid'], [int(time.time()), None], 'backup', '%d-%s' % (backup_timestamp, dle_id)) if rc != ade.ok: return rc return ade.ok def purge_stale_backup_locks(errstack, cursor): ade.assert_inside_sql_transaction(errstack) sql_statement = ''' SELECT backup_timestamp, dle_id, lock_pid FROM backups WHERE lock_pid IS NOT NULL; ''' sql_value = () rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # ... and filter them down the list that are not actually running ... stalely_locked_backups = [ (backup_timestamp,dle_id) for (backup_timestamp,dle_id,lock_pid) in select_results if not os.path.isdir('/proc/%d' % (lock_pid)) ] # ... and unlock those. for stalely_locked_backup in stalely_locked_backups: rc = unlock_backup(errstack, cursor, stalely_locked_backup[0], stalely_locked_backup[1]) if rc != ade.ok: return rc return ade.ok def lock_instance(errstack, cursor, pid, db_access_mode, this_command): ade.debug(errstack, 10, 'lock_instance: registering this process ...') rc = sql_lock_row(errstack, cursor, 'locks', 'insert', None, None, ['lock_pid', 'db_access_mode', 'cmd', 'start_timestamp'], [pid, db_access_mode, this_command, int(time.time())], 'pid', pid, lambda x : multireplace(x, {'ConstraintError: ':''})) if rc != ade.ok: return rc return ade.ok def unlock_instance(errstack, cursor, pid): ade.assert_inside_sql_transaction(errstack) rc = sql_unlock_row(errstack, cursor, 'locks', 'delete', ['lock_pid'], [pid], None, None, 'pid', pid, lambda x : multireplace(x, {'ConstraintError: ':''})) if rc != ade.ok: return rc return ade.ok def purge_stale_instance_locks(errstack, cursor): ade.assert_inside_sql_transaction(errstack) sql_statement = ''' SELECT lock_pid FROM locks WHERE lock_pid IS NOT NULL; ''' sql_value = () rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # ... and filter them down the list that are not actually running ... stalely_locked_pids = [ lock_pid for (lock_pid,) in select_results if not os.path.isdir('/proc/%d' % (lock_pid)) ] # ... and unlock those. for stalely_locked_pid in stalely_locked_pids: rc = unlock_instance(errstack, cursor, stalely_locked_pid) if rc != ade.ok: return rc return ade.ok ######################################################################## # # Other helpers # ######################################################################## def cache_medium_accesspoints(errstack, cursor): global medium_accesspoints_cached_flag ade.debug(errstack, 10, 'cache_medium_accesspoints: checking cache not valid yet ...') if medium_accesspoints_cached_flag is not False: ade.internal(errstack, 'cache_medium_accesspoints: cached already! (hint: cache/uncache calls balanced?)') ade.debug(errstack, 10, 'cache_medium_accesspoints: creating cache table ...') sql_statement = ''' CREATE TEMP TABLE medium_accesspoints ( medium_label TEXT NOT NULL, medium_accesspoint TEXT NOT NULL, PRIMARY KEY (medium_label), -- Really we want to make the following foreign key constraint: -- -- FOREIGN KEY (medium_label) REFERENCES main.media(medium_label), -- -- But this is not possible because *another process* could -- alter the 'media' table (e.g. by deleting a row from it) -- and, in the process, violate the foreign key constraint. -- See https://sqlite-users.sqlite.narkive.com/m6g9nMKH/. UNIQUE(medium_accesspoint) ); ''' ade.execute_sql(errstack, cursor, sql_statement) # Get inspector commands. ade.debug(errstack, 10, 'cache_medium_accesspoints: getting list of medium accesspoint inspectors ...') sql_statement = ''' SELECT accesspoint_inspector_method_cmd FROM accesspoint_inspector_methods, servers WHERE 0 < (SELECT COUNT(*) FROM accesspoint_inspector_method_grants WHERE accesspoint_inspector_method_grants.accesspoint_inspector_method_id = accesspoint_inspector_methods.accesspoint_inspector_method_id AND (accesspoint_inspector_method_grants.server_type_id IS NULL OR accesspoint_inspector_method_grants.server_type_id = servers.server_type_id)) ORDER BY accesspoint_inspector_methods.accesspoint_inspector_method_id; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc accesspoint_inspector_method_cmds = [r[0] for r in select_results] ade.debug(errstack, 10, 'cache_medium_accesspoints: medium accesspoint inspectors are: %s' % (', '.join(accesspoint_inspector_method_cmds))) # Run inspector commands and insert results into cache. for accesspoint_inspector_method_cmd in accesspoint_inspector_method_cmds: ade.debug(errstack, 10, 'cache_medium_accesspoints: calling accesspoint inspector %s ...' % (accesspoint_inspector_method_cmd)) rc, abs_accesspoint_inspector_method_cmd = resolve_if_exists(errstack, 'methods/' + accesspoint_inspector_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_accesspoint_inspector_method_cmd, '--debug=%d' % verboselevel] ade.debug(errstack, 10, 'cache_medium_accesspoints: command is %s' % (cmdline)) try: # Don't redirect inspector's stderr. child = subprocess.Popen(cmdline, shell=False, stdout=subprocess.PIPE, universal_newlines=True) except FileNotFoundError: ade.internal(errstack, 'cache_medium_accesspoints: %s: failed to execute' % (cmdline[0])) stdout = child.communicate()[0] rc = child.wait() ade.debug(errstack, 10, 'cache_medium_accesspoints: stdout=%s, rc=%d' % (stdout, rc)) if rc != 0: ade.debug(errstack, 10, 'cache_medium_accesspoints: accesspoint inspector failed; returning error ...') ade.error(errstack, 'rdw2_err_misc', 'accesspoint inspector returned non-zero (hint: see error messages above)') return ade.fail # The original SQL I used to insert the medium label and access # point into medium_accesspoints was simply: # # INSERT INTO medium_accesspoints VALUES (?, ?); # # but the problem with that is that it does not take account of # the medium being disabled in the media table. The following # SQL does, though note that the two '?' are reversed (i.e. # above sql_value was (medium_label, medium_accesspoint) but # now it is (medium_accesspoint, medium_label). That's just due # to the change in the order in which the two column names appear # in the following statement. sql_statement = ''' INSERT INTO medium_accesspoints SELECT media.medium_label, ? AS medium_accesspoint FROM media WHERE media.medium_label = ? AND media.enabled = 'true'; ''' for medium_label,medium_accesspoint in [x.split(' ') for x in stdout.splitlines()]: sql_value = (medium_accesspoint, medium_label) ade.debug(errstack, 10, 'cache_medium_accesspoints: inserting %s into cache using SQL \'%s\' ...' % (sql_value, sql_statement)) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc ade.debug(errstack, 10, 'cache_medium_accesspoints: marking cache valid ...') medium_accesspoints_cached_flag = True return ade.ok def uncache_medium_accesspoints(errstack, cursor): global medium_accesspoints_cached_flag if medium_accesspoints_cached_flag is False: ade.internal(errstack, 'cache_medium_accesspoints: uncached already! (hint: cache/uncache calls balanced?)') sql_statement = ''' DROP TABLE medium_accesspoints; ''' ade.execute_sql(errstack, cursor, sql_statement) medium_accesspoints_cached_flag = False return ade.ok def get_medium_accesspoint(errstack, cursor, medium_id): # # Map medium_id to medium_type_id # sql_statement = ''' # SELECT medium_type_id # FROM media # WHERE medium_id = ?; # ''' # sql_value = (medium_id,) # rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) # if rc != ade.ok: # return rc, None # if len(select_results) == 0: # ade.error(errstack, 'rdw2_err_misc', '%s: invalid medium (hint: to see valid media run \'%s --list-media\')') # return ade.fail, None # elif len(select_results) > 1: # ade.internal(errstack, 'get_medium_accesspoint: %s: multiple media with this medium id found!') # (medium_type_id,) = select_results[0] # Get accesspoints of all media of that type. rc = cache_medium_accesspoints(errstack, cursor) if rc != ade.ok: return rc, None sql_statement = ''' SELECT medium_accesspoint FROM medium_accesspoints, media WHERE medium_accesspoints.medium_label = media.medium_label AND medium_id = ?; ''' sql_value = (medium_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc, None if len(select_results) == 0: ade.error(errstack, 'rdw2_err_misc', '%s: could not locate (hint: is it mounted/inserted?)' % (medium_id)) return ade.fail, None return ade.ok, select_results[0][0] def get_preferred_set_medium_and_its_accesspoint(errstack, cursor, set_id): rc = cache_medium_accesspoints(errstack, cursor) if rc != ade.ok: return rc, None, None sql_statement = ''' SELECT media.medium_id, medium_accesspoints.medium_accesspoint FROM media, medium_accesspoints, medium_set_compatibility WHERE media.medium_label = medium_accesspoints.medium_label AND medium_set_compatibility.medium_id = media.medium_id AND -- The above conditions are straightforward requirements for -- joining the three tables. The ones below restrict results -- to media that are about the specificed set and are -- compatible with it. medium_set_compatibility.compatible = 1 AND set_id = ? -- Finally we select the oldest of the results so as to -- force rotating media correctly. ORDER BY medium_last_used_timestamp ASC LIMIT 1; ''' sql_value = (set_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc, None, None # Check that we didn't get no rows back. if len(select_results) == 0: ade.error(errstack, 'rdw2_err_misc', 'no suitable medium is accessible') return ade.fail, None, None #for select_result in select_results: # (medium_id,) = select_result # ade.debug(errstack, 10, 'get_preferred_set_medium_and_its_accesspoint: medium_id=%s' % (medium_id)) return ade.ok, select_results[0][0], select_results[0][1] def get_preferred_dle_medium_and_its_accesspoint(errstack, cursor, dle_id): # We don't want DLEs to spread onto different media to other # DLEs in the same set. So we say that the the media that are # suitable for a particular dle are those that are suitable # for the set that contains that dle. So map the DLE back to # the containing set and ... sql_statement = ''' SELECT set_id FROM dles WHERE dle_id = ?;''' sql_value = (dle_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc, None, None if len(select_results) == 0: ade.error(errstack, 'rdw2_err_misc', '%s: invalid DLE (hint: to see valid DLEs run \'%s --list-dles\')') return ade.fail, None, None (set_id,) = select_results[0] # ... then look up the preferred set medium. return get_preferred_set_medium_and_its_accesspoint(errstack, cursor, set_id) def validate_ssh(errstack, hostname): ade.debug(errstack, 10, 'validate_ssh: %s: validating ...' % (hostname)) ssh_output = re.sub('\n', '; ', subprocess.Popen(['ssh', '-n', '-oStrictHostKeyChecking=no', hostname, 'echo OK'], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True).communicate()[0].rstrip('\n')) ade.debug(errstack, 10, 'validate_ssh: ssh_output=%s' % (ssh_output)) if ssh_output != 'OK': # Display message if 40 chars or less else display 37 chars + '...'. ade.error(errstack, 'rdw2_err_misc', '%s: ssh access check failed (output was: %s)' % (hostname, '%s' if len(ssh_output) <= 40 else '%s...' % (ssh_output[:37]))) return ade.fail return ade.ok # This function will be referenced by a new function '--add-schedule', which has yet to be written. def validate_schedule(errstack, schedule): ade.debug(errstack, 10, 'validate_schedule: %s: validating ...' % (schedule)) regexps = { 'minute': '^(?:[0-9]|[0-5][0-9])$', 'hour': '^(?:[0-9]|[0-1][0-9]|2[0-3])$', 'dom': '^(?:0?[1-9]|[12][0-9]|3[0-1])$', 'month': '^(?:0?[1-9]|1[0-2])$', 'dow': '^([0-6])$' } m = re.search('^([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+)$', schedule) if m is None: ade.error(errstack, 'rdw2_err_misc', '%s: not a valid schedule' % (schedule)) return ade.fail, None for (value, validator_regexp, field_name) in [(m.group(1), regexps['minute'], 'minutes'), (m.group(2), regexps['hour'], 'hours'), (m.group(3), regexps['dom'], 'day of month'), (m.group(4), regexps['month'], 'month'), (m.group(5), regexps['dow'], 'day of week')]: ade.debug(errstack, 10, 'validate_schedule: value=%s, validator_regexp=%s, field_name=%s' % (value, validator_regexp,field_name)) for comma_separated_value in value.split(','): ade.debug(errstack, 10, 'validate_schedule: comma_separated_value=%s' % (comma_separated_value)) m = re.search('(.*)/[1-9][0-9]*$', comma_separated_value) if m is not None: demoduloed_comma_separated_value = m.group(1) else: demoduloed_comma_separated_value = comma_separated_value ade.debug(errstack, 10, 'validate_schedule: demoduloed_comma_separated_value=%s' % (demoduloed_comma_separated_value)) if demoduloed_comma_separated_value != '*' and not re.search(validator_regexp, demoduloed_comma_separated_value): ade.error(errstack, 'rdw2_err_misc', '%s: not a valid %s field of a schedule' % (value, field_name)) return ade.fail, None return ade.ok def validate_server(errstack, cursor): ade.debug(errstack, 10, 'validate_server: validating current server ...') # No question mark 'cos there is only one. sql_statement = ''' SELECT server_hostname, (SELECT server_labelling_method_cmd FROM server_labelling_methods WHERE server_labelling_methods.server_labelling_method_id=servers.server_labelling_method_id) AS server_labelling_method_cmd, server_label FROM servers; ''' rc, select_results = ade.select_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc if len(select_results) != 1: ade.internal(errstack, 'validate_server: %s: %d matching servers were found' % (server_id, select_count)) (server_hostname, server_labelling_method_cmd, server_label) = select_results[0] ade.debug(errstack, 10, 'validate_server: database says server_hostname=%s, server_labelling_method_cmd=%s, server_label=%s' % (server_hostname, server_labelling_method_cmd, server_label)) # Verify server's label ade.debug(errstack, 10, 'validate_server: verifying %s ...' % (server_hostname)) rc, abs_server_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + server_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_server_labelling_method_cmd, '--debug=%d' % verboselevel, 'verify', server_label] ade.debug(errstack, 10, 'validate_server: command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'validate_server: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'server labeller returned non-zero (hint: see error messages above)') return ade.fail return ade.ok def validate_client(errstack, cursor, client_id): ade.debug(errstack, 10, 'validate_client: %s: validating ...' % (client_id)) sql_statement = ''' SELECT client_hostname, (SELECT client_labelling_method_cmd FROM client_labelling_methods WHERE client_labelling_methods.client_labelling_method_id=clients.client_labelling_method_id) AS client_labelling_method_cmd, client_label FROM clients WHERE client_id = ?; ''' sql_value = (client_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if len(select_results) == 0: ade.error(errstack, 'rdw2_err_misc', '%s: no such client' % (client_id)) return ade.fail elif len(select_results) > 1: ade.internal(errstack, 'validate_client: %s: %d matching clients were found' % (client_id, select_count)) (client_hostname, client_labelling_method_cmd, client_label) = select_results[0] ade.debug(errstack, 10, 'validate_client: database says client_hostname=%s, client_labelling_method_cmd=%s, client_label=%s' % (client_hostname, client_labelling_method_cmd, client_label)) rc = validate_ssh(errstack, client_hostname) if rc != ade.ok: return rc # Verify client's label ade.debug(errstack, 10, 'validate_client: %s: verifying label ...' % (client_id)) rc, abs_client_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + client_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_client_labelling_method_cmd, '--debug=%d' % verboselevel, 'verify', client_hostname, client_label] ade.debug(errstack, 10, 'validate_client: %s: command is %s' % (client_id, cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'validate_client: %s: %s: command not found' % (client_id, cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', '%s: label verifier returned non-zero (hint: see error messages above)' % (client_id)) return ade.fail ade.debug(errstack, 10, 'validate_client: %s: validated' % (client_id)) return ade.ok ######################################################################## # # Minor support functions # ######################################################################## def load_config(errstack): global rdw2_cmd, rdw2_server_flag actual_configvars = {} allowed_configvars = ['rdw2_cmd', 'rdw2_server_flag'] config_file = '%s/%s' % (rdw2_etc_prefix, 'rdw2.py') try: exec(open(config_file).read(), None, actual_configvars) except FileNotFoundError: pass except: ade.error(errstack, 'rdw2_err_misc', '%s: failed to load config file (hint: is it valid python?)' % (config_file)) return ade.fail, actual_configvars # Defaults rdw2_cmd = None rdw2_server_flag = False for key in actual_configvars.keys(): if key == 'rdw2_cmd': rdw2_cmd = actual_configvars['rdw2_cmd'] elif key == 'rdw2_server_flag': if not isinstance(actual_configvars['rdw2_server_flag'], bool): ade.error(errstack, 'rdw2_err_misc', 'rdw2_server_flag: has invalid value (hint: should be \'True\' or \'False\' or not set at all)') return ade.fail rdw2_server_flag = actual_configvars['rdw2_server_flag'] elif key == 'rdw2_share_prefix': if not isinstance(actual_configvars['rdw2_share_prefix'], str): ade.error(errstack, 'rdw2_err_misc', 'rdw2_share_prefix: has invalid value (hint: should be string)') return ade.fail rdw2_share_prefix = actual_configvars['rdw2_share_prefix'] else: ade.error(errstack, 'rdw2_err_misc', '%s: invalid setting' % (key)) return ade.fail return ade.ok def reexecutor(errstack): global rdw2_cmd if rdw2_cmd is None: pass elif 'RDW2_RELOADED_FLAG' in os.environ: del os.environ['RDW2_RELOADED_FLAG'] else: ade.debug(errstack, 10, 'reexecutor: reexecuting ...') os.environ['RDW2_RELOADED_FLAG'] = 'true' try: os.execv(rdw2_cmd, ['rdw2'] + sys.argv[1:]) except: ade.error(errstack, 'rdw2_err_misc', 're-execution error (hint: is rdw2_cmd set correctly?)') return ade.fail return ade.ok def s2ms(s): return '%02d:%02d' % (s/60, s%60) def line(c): sys.stdout.write('%s\n' % (c * 100)) def multireplace(s, m): for k in m.keys(): s=s.replace(k,m[k]) return s def check_dle_exists(errstack, cursor, dle_id): sql_statement = ''' SELECT COUNT(*) FROM dles WHERE dle_id = ?; ''' sql_value = (dle_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: dle does not exist' % (dle_id)) return ade.fail return ade.ok def check_set_exists(errstack, cursor, set_id): sql_statement = ''' SELECT COUNT(*) FROM sets WHERE set_id = ?; ''' sql_value = (set_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: set does not exist' % (set_id)) return ade.fail return ade.ok def check_backup_method_exists(errstack, cursor, backup_method_id): sql_statement = ''' SELECT COUNT(*) FROM backup_methods WHERE backup_method_id = ?; ''' sql_value = (backup_method_id,) rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc if select_count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: backup method does not exist' % (backup_method_id)) return ade.fail return ade.ok def check_file_readable(errstack, thing): if not (os.path.isfile(thing) and os.access(thing, os.R_OK)): ade.error(errstack, 'rdw2_err_misc', '%s: is not a readable file' % (thing)) return ade.fail return ade.ok def check_file_writable(errstack, thing): if (os.path.isfile(thing) and not os.access(thing, os.W_OK)) or not os.access(thing.rsplit('/', 1)[0], os.W_OK): ade.error(errstack, 'rdw2_err_misc', '%s: is not a writable file' % (thing)) return ade.fail return ade.ok def validate_or_add_this_server(errstack, cursor): ade.debug(errstack, 10, 'validate_or_add_this_server: calling standard_prologue() ...') rc, progname = ade.get_progname(errstack) rc = standard_prologue(errstack, cursor, False, False, 'shared', '%s --fixme' % (progname)) if rc != ade.ok: return rc ade.debug(errstack, 10, 'validate_or_add_this_server: checking if database has a label for this server ...') server_id = socket.gethostname() sql_value = (server_id,) sql_statement = ''' SELECT COUNT(*) FROM servers WHERE server_id = ?; ''' rc, select_count = ade.select_sql_count_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # If the server is in the database, then we just need to validate its label, finishing the # transaction regardless of whether validation succeeds or fails. if select_count == 1: ade.debug(errstack, 10, 'validate_or_add_this_server: database has a label for this server; validating server ...') rc = validate_server(errstack, cursor) # The transaction ends after we've (successfully or unsuccessfully) validated the server. # Note use of rc2: we need to keep rc to determine if we need to put error on stack. rc2 = standard_epilogue(errstack, cursor, False) if rc2 != ade.ok: return rc2 if rc != ade.ok: return rc return ade.ok # If we get to here then the server is not in the database. So we should attempt to label it. # Collect all the information we need (OS, FQHN, new label, labelling method). ade.debug(errstack, 10, 'validate_or_add_this_server: database does not have a label for this server; preparing to label ...') server_hostname = socket.getfqdn() server_type_id = platform.system() if server_type_id != 'Linux': ade.internal(errstack, 'validate_or_add_this_server: %s: unhandled OS' % (server_type_id)) server_labelling_method_id = 'df-in-sr' rc, server_label = ade.create_uuid(errstack) if rc != ade.ok: return rc sql_statement = ''' SELECT server_labelling_method_cmd FROM server_labelling_methods WHERE server_labelling_method_id = ?; ''' sql_value = (server_labelling_method_id,) rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (server_labelling_method_cmd,) = select_results[0] # Then do the insert, which is still inside a transaction so rollable-backable. ade.debug(errstack, 10, 'validate_or_add_this_server: writing new label to database ...') sql_statement = ''' INSERT INTO servers VALUES (?, ?, ?, ?, ?, ?); ''' sql_value = (server_id, 'true', server_type_id, server_hostname, server_labelling_method_id, server_label) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc # The insert was successful but not committed. Any failures from here on should do a rollback # (possibly simply by returning an error before getting to the commit code). # Do the labelling. ade.debug(errstack, 10, 'validate_or_add_this_server: writing new label to disk ...') rc, abs_server_labelling_method_cmd = resolve_if_exists(errstack, 'methods/' + server_labelling_method_cmd, [ rdw2_etc_prefix, rdw2_share_prefix ]) if rc != ade.ok: return rc rc, verboselevel = ade.get_verboselevel(errstack) cmdline = [abs_server_labelling_method_cmd, '--debug=%d' % verboselevel, 'add', server_label] ade.debug(errstack, 10, 'put_server: command is %s' % (cmdline)) try: # Don't redirect method's stdout and stderr. child = subprocess.Popen(cmdline, shell=False) except FileNotFoundError: ade.internal(errstack, 'put_server: %s: failed to execute' % (cmdline[0])) rc = child.wait() if rc != 0: ade.error(errstack, 'rdw2_err_misc', 'server is already labelled or labelling failed for another reason (hint: see error messages above)') return ade.fail # Commit. ade.debug(errstack, 10, 'validate_or_add_this_server: committing changes to database ...') rc = standard_epilogue(errstack, cursor, False) if rc != ade.ok: return rc return ade.ok def upgrade_from_0_to_1(errstack, cursor): ade.error(errstack, 'rdw2_err_misc', 'there never was a database with conformancy 0 (hint: have you blanked the database?)') return ade.fail def upgrade_from_1_to_2(errstack, cursor): sql_statement = ''' CREATE TABLE backup_filters ( dle_id CHAR NOT NULL, backup_method_id CHAR NOT NULL, enabled CHAR NOT NULL, backup_filter BLOB, PRIMARY KEY (dle_id, backup_method_id), FOREIGN KEY (dle_id) REFERENCES dles(dle_id), FOREIGN KEY (backup_method_id) REFERENCES backup_methods(backup_method_id), FOREIGN KEY (enabled) REFERENCES logicals(logical) ); ''' rc = ade.execute_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc ade.set_sql_schema_version(errstack, cursor, 2) return ade.ok def upgrade_from_2_to_3(errstack, cursor): sql_statements = [ # SQL doesn't allow to modify constraints so we just create a new table, # copy over the data, drop the old table, rename the new one. ''' create table backup_filters_new ( dle_id CHAR, backup_method_id CHAR, -- dle_id+backup_method_id would be primary key but that doesn't support NULL as -- a unique value, so we need to implement the constraint in a trigger. See the -- backup_filters_pre_insert trigger below. enabled CHAR NOT NULL, backup_filter BLOB, FOREIGN KEY (dle_id) REFERENCES dles(dle_id), FOREIGN KEY (backup_method_id) REFERENCES backup_methods(backup_method_id), FOREIGN KEY (enabled) REFERENCES logicals(logical) ); ''', ''' INSERT INTO backup_filters_new (dle_id, backup_method_id, enabled, backup_filter) SELECT dle_id, backup_method_id, enabled, backup_filter FROM backup_filters; ''', ''' DROP TABLE backup_filters; ''', ''' ALTER TABLE backup_filters_new RENAME TO backup_filters; ''', ''' CREATE TRIGGER backup_filters_pre_insert BEFORE INSERT ON backup_filters BEGIN SELECT RAISE(FAIL, "%{dle_id},%{backup_method_id}: a filter for combination is already defined") WHERE ( SELECT COUNT(*) FROM backup_filters WHERE (dle_id = NEW.dle_id OR (dle_id IS NULL AND NEW.dle_id IS NULL)) AND (backup_method_id = NEW.backup_method_id OR (backup_method_id IS NULL AND NEW.backup_method_id IS NULL)) ) == 1; END ; ''' ] for sql_statement in sql_statements: rc = ade.execute_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc ade.set_sql_schema_version(errstack, cursor, 3) return ade.ok def upgrade_from_3_to_4(errstack, cursor): sql_statements = [ # Comments are omitted from this SQL, but they are in the # .sql file. # # medium_grants table had first column medium_labelling_method_grant_id, # which is wrong (presumably a copy-and-paste error). We correct # that here. ''' CREATE TABLE medium_grants_new ( -- columns medium_grant_id INTEGER, medium_id CHAR, set_id CHAR, dle_id CHAR, client_id CHAR, server_id CHAR, -- clauses PRIMARY KEY (medium_grant_id), FOREIGN KEY (medium_id) REFERENCES media(medium_id), FOREIGN KEY (set_id) REFERENCES sets(set_id), FOREIGN KEY (dle_id) REFERENCES dles(dle_id), FOREIGN KEY (client_id) REFERENCES clients(client_id), FOREIGN KEY (server_id) REFERENCES servers(server_id) ); ''', ''' INSERT INTO medium_grants_new (medium_grant_id, medium_id, set_id, dle_id, client_id, server_id) SELECT medium_labelling_method_grant_id, medium_id, set_id, dle_id, client_id, server_id FROM medium_grants; ''', ''' DROP TABLE medium_grants; ''', ''' ALTER TABLE medium_grants_new RENAME TO medium_grants; ''', # New view set_media. ''' CREATE VIEW set_media AS SELECT dles.set_id, media.medium_id, DATETIME(COALESCE((SELECT MIN(backup_timestamp) FROM backups WHERE backups.medium_id = media.medium_id),0), 'unixepoch', 'localtime') AS medium_last_used_timestamp FROM media, dles, servers WHERE media.enabled = 'true' AND 0 < (SELECT COUNT(*) FROM medium_grants WHERE medium_grants.medium_id = media.medium_id AND (medium_grants.set_id IS NULL OR medium_grants.set_id = dles.set_id) AND (medium_grants.dle_id IS NULL OR medium_grants.dle_id = dles.dle_id) AND (medium_grants.client_id IS NULL OR medium_grants.client_id = dles.client_id) AND (medium_grants.server_id IS NULL OR medium_grants.server_id = servers.server_id)) GROUP BY media.medium_id HAVING COUNT(*) = (SELECT COUNT(*) FROM dles AS d2 WHERE d2.set_id = dles.set_id) ORDER BY set_id, medium_last_used_timestamp; ''', ] for sql_statement in sql_statements: rc = ade.execute_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc ade.set_sql_schema_version(errstack, cursor, 4) return ade.ok def upgrade_from_4_to_5(errstack, cursor): sql_statements = [ # Comments are omitted from this SQL, but they are in the # .sql file. ''' INSERT INTO backup_methods VALUES ('rsync-over-btrfs', 'true', 'false', 'Linux/Linux/backups/rsync-over-btrfs'); INSERT INTO backup_methods VALUES ('rsync-over-btrfs-one-fs', 'true', 'false', 'Linux/Linux/backups/rsync-over-btrfs-one-fs'); INSERT INTO backup_method_grants VALUES (NULL,'rsync-over-btrfs','Linux','Linux','floating-dir','dir'); INSERT INTO backup_method_grants VALUES (NULL,'rsync-over-btrfs-one-fs','Linux','Linux','floating-dir','dir-one-fs'); INSERT INTO dle_labelling_methods VALUES('row-in-sqlite-db','true','Linux/Linux/dle-labellers/row-in-sqlite-db'); INSERT INTO dle_labelling_method_grants VALUES(NULL,'row-in-sqlite-db','Linux','Linux','dir'); INSERT INTO dle_labelling_method_grants VALUES(NULL,'row-in-sqlite-db','Linux','Linux','dir-one-fs'); DROP VIEW set_media; CREATE VIEW medium_dle_applicable_grants AS SELECT media.medium_id, dles.dle_id, medium_grants.grant_id FROM media, dles, servers, medium_grants WHERE medium_grants.medium_id = media.medium_id AND (medium_grants.set_id IS NULL OR medium_grants.set_id = dles.set_id) AND (medium_grants.dle_id IS NULL OR medium_grants.dle_id = dles.dle_id) AND (medium_grants.client_id IS NULL OR medium_grants.client_id = dles.client_id) AND (medium_grants.server_id IS NULL OR medium_grants.server_id = servers.server_id); CREATE VIEW medium_dle_compatibility AS SELECT media.medium_id, dles.dle_id, ( SELECT COUNT(*)>0 FROM medium_dle_applicable_grants WHERE media.medium_id=medium_dle_applicable_grants.medium_id AND dles.dle_id=medium_dle_applicable_grants.dle_id ) AS compatible, DATETIME(COALESCE( (SELECT MIN(backup_timestamp) FROM backups WHERE backups.medium_id = media.medium_id), 0), 'unixepoch', 'localtime' ) AS medium_last_used_timestamp FROM media, dles; CREATE VIEW medium_set_compatibility AS SELECT set_id, medium_id, ( SELECT COUNT(*) FROM dles WHERE dles.set_id=sets.set_id ) = COALESCE( (SELECT SUM(compatible) FROM dles, medium_dle_compatibility WHERE dles.dle_id = medium_dle_compatibility.dle_id AND dles.set_id = sets.set_id AND medium_dle_compatibility.medium_id = media.medium_id),0 ) AS compatible, DATETIME(COALESCE( (SELECT MIN(backup_timestamp) FROM backups WHERE backups.medium_id = media.medium_id), 0), 'unixepoch', 'localtime' ) AS medium_last_used_timestamp FROM sets, media; ''' ] for sql_statement in sql_statements: rc = ade.execute_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc ade.set_sql_schema_version(errstack, cursor, 5) return ade.ok def upgrade_from_5_to_6(errstack, cursor): sql_statements = [ # Comments are omitted from this SQL, but they are in the # .sql file. # # View medium_dle_applicable_grants refered to medium_grants.grant_id, # but the correct name is actually medium_grants.medium_grant_id. We # fix that here. ''' DROP VIEW medium_dle_applicable_grants; ''', ''' CREATE VIEW medium_dle_applicable_grants AS SELECT media.medium_id, dles.dle_id, medium_grants.medium_grant_id FROM media, dles, servers, medium_grants WHERE medium_grants.medium_id = media.medium_id AND (medium_grants.set_id IS NULL OR medium_grants.set_id = dles.set_id) AND (medium_grants.dle_id IS NULL OR medium_grants.dle_id = dles.dle_id) AND (medium_grants.client_id IS NULL OR medium_grants.client_id = dles.client_id) AND (medium_grants.server_id IS NULL OR medium_grants.server_id = servers.server_id); ''', # The backup_filters table contained *multiple* filters # per row (newline-packed) but that has changed to single # filters per row. This has resulted in primary key changes. # We drop and recreate the table. We preserve data but we # do not newline-unpack it, so the filters themselve will # absolutely need fixing manually. Additionally, there is # an associated pre-insert trigger that needs updating. # Again, we delete and recreate this. ''' DROP TRIGGER backup_filters_pre_insert; ''', ''' CREATE TABLE backup_filters_new ( backup_filter CHAR NOT NULL, set_id CHAR, dle_id CHAR, backup_method_id CHAR, enabled CHAR NOT NULL, FOREIGN KEY (set_id) REFERENCES sets(set_id), FOREIGN KEY (dle_id) REFERENCES dles(dle_id), FOREIGN KEY (backup_method_id) REFERENCES backup_methods(backup_method_id), FOREIGN KEY (enabled) REFERENCES logicals(logical) ); ''', ''' INSERT INTO backup_filters_new (backup_filter, set_id, dle_id, backup_method_id, enabled) SELECT backup_filter, NULL, dle_id, backup_method_id, enabled FROM backup_filters; ''', ''' DROP TABLE backup_filters; ''', ''' ALTER TABLE backup_filters_new RENAME TO backup_filters; ''', ''' CREATE TRIGGER backup_filters_pre_insert BEFORE INSERT ON backup_filters BEGIN SELECT RAISE(FAIL, "%{BACKUP_FILTER},%{SET_ID},%{DLE_ID},%{BACKUP_METHOD_ID}: a filter for combination is already defined") WHERE ( SELECT COUNT(*) FROM backup_filters WHERE (backup_filter = NEW.backup_filter) AND (set_id = NEW.set_id OR (set_id IS NULL AND NEW.set_id IS NULL)) AND (dle_id = NEW.dle_id OR (dle_id IS NULL AND NEW.dle_id IS NULL)) AND (backup_method_id = NEW.backup_method_id OR (backup_method_id IS NULL AND NEW.backup_method_id IS NULL)) ) == 1; END ; ''', ] for sql_statement in sql_statements: rc = ade.execute_sql(errstack, cursor, sql_statement) if rc != ade.ok: return rc ade.set_sql_schema_version(errstack, cursor, 6) return ade.ok def resolve_if_exists(errstack, relative_path, absolute_prefixes): ade.debug(errstack, 10, 'resolve_if_exists: checking if %s is in any of %s ...' % (relative_path, ', '.join(absolute_prefixes))) found = False for absolute_prefix in absolute_prefixes: absolute_path = absolute_prefix + '/' + relative_path if os.path.exists(absolute_path): found = True break if not found: ade.debug(errstack, 10, 'resolve_if_exists: not found') ade.error(errstack, 'rdw2_err_misc', '%s: not found' % (relative_path)) return ade.fail, None ade.debug(errstack, 10, 'resolve_if_exists: found at %s' % (absolute_path)) return ade.ok, absolute_path #def save_old_dir_contents(errstack, cp_or_mv, src): # global work_dir # # if cp_or_mv not in ['cp', 'mv']: # ade.internal(errstack, 'save_old_dir_contents: %s: bad value for cp_or_mv' % (cp_or_mv)) # rc, progname = ade.get_progname(errstack) # tgt = '%s/%s.%d.save' % (work_dir, progname, os.getpid()) # # # Guts # ade.warning(errstack, 'rdw2_err_misc', 'saving old %s to %s ...' % (src, tgt)) # if cp_or_mv == 'mv' and os.path.isfile(src): # os.mkdir(tgt) # shutil.move(src, tgt) # elif cp_or_mv == 'mv' and os.path.isdir(src): # shutil.move(src, tgt) # elif cp_or_mv == 'cp' and os.path.isfile(src): # os.mkdir(tgt) # shutil.copy(src, tgt) # elif cp_or_mv == 'cp' and os.path.isdir(src): # shutil.copytree(src,tgt) # else: # ade.internal(errstack, 'save_old_dir_contents: %s: not a file or a directory' % (src)) # # return ade.ok ######################################################################## # # Standard ADE-compatibility functions # ######################################################################## def rdw2_version(errstack): return ade.extract_version(errstack, app_svnid) def rdw2_paths(errstack): global code_schema_conformancy, opt_db_file return ade.ok, '\n'.join([ \ 'Db-File: %s' % (opt_db_file), \ ]) def rdw2_usage_help(errstack): return ade.ok, None, ' --edit\n' + \ ' --upgrade\n' + \ ' --add-set \n' + \ ' --del-set \n' + \ ' --list-sets [ ... ]\n' + \ ' --add-client \\\n' + \ ' \n' + \ ' --del-client \n' + \ ' --list-clients [ ... ]\n' + \ ' --list-servers [ ... ]\n' + \ ' --add-medium \\\n' + \ ' \n' + \ ' --del-medium \n' + \ ' --list-media [ ... ]\n' + \ ' --list-set-media \n' + \ ' --list-dle-media \n' + \ ' --list-preferred-set-medium \n' + \ ' --list-preferred-dle-medium \n' + \ ' --add-dle \\\n' + \ ' \\\n' + \ ' \n' + \ ' --del-dle \n' + \ ' --list-dles [ ... ]\n' + \ ' --backup-dle \n' + \ ' --check-dle \n' + \ ' --backup-set \n' + \ ' --check-set \n' + \ ' --list-backups [ ... ]\n' + \ ' --list-misc [ ... ]\n' + \ ' --list-medium-accesspoints\n' + \ ' --list-medium-accesspoint \n' + \ ' --list-pids [ ... ]\n' + \ ' --list-all [ ... ]\n' + \ ' --add-filter { | \'\' } { | \'\' }\n' + \ ' { | \'\' }\n' + \ ' --del-filter { | \'\' } { | \'\' }\n' + \ ' { | \'\' }\n' + \ ' --list-filters\n' + \ ' --no-update-cron\n' + \ ' --db-file=\n' return ade.ok def rdw2_opt_handler_upgrade(errstack): global opt_mode opt_mode = 'upgrade' return ade.ok def rdw2_opt_handler_edit(errstack): global opt_mode opt_mode = 'edit' return ade.ok def rdw2_opt_handler_list_sets(errstack): global opt_mode opt_mode = 'list-sets' return ade.ok def rdw2_opt_handler_add_set(errstack): global opt_mode opt_mode = 'add-set' return ade.ok def rdw2_opt_handler_del_set(errstack): global opt_mode opt_mode = 'del-set' return ade.ok def rdw2_opt_handler_list_clients(errstack): global opt_mode opt_mode = 'list-clients' return ade.ok def rdw2_opt_handler_add_client(errstack): global opt_mode opt_mode = 'add-client' return ade.ok def rdw2_opt_handler_del_client(errstack): global opt_mode opt_mode = 'del-client' return ade.ok def rdw2_opt_handler_list_servers(errstack): global opt_mode opt_mode = 'list-servers' return ade.ok def rdw2_opt_handler_list_media(errstack): global opt_mode opt_mode = 'list-media' return ade.ok def rdw2_opt_handler_list_dle_media(errstack): global opt_mode opt_mode = 'list-dle-media' return ade.ok def rdw2_opt_handler_list_set_media(errstack): global opt_mode opt_mode = 'list-set-media' return ade.ok def rdw2_opt_handler_list_preferred_dle_medium(errstack): global opt_mode opt_mode = 'list-preferred-dle-medium' return ade.ok def rdw2_opt_handler_list_preferred_set_medium(errstack): global opt_mode opt_mode = 'list-preferred-set-medium' return ade.ok def rdw2_opt_handler_add_medium(errstack): global opt_mode opt_mode = 'add-medium' return ade.ok def rdw2_opt_handler_del_medium(errstack): global opt_mode opt_mode = 'del-medium' return ade.ok def rdw2_opt_handler_list_dles(errstack): global opt_mode opt_mode = 'list-dles' return ade.ok def rdw2_opt_handler_add_dle(errstack): global opt_mode opt_mode = 'add-dle' return ade.ok def rdw2_opt_handler_del_dle(errstack): global opt_mode opt_mode = 'del-dle' return ade.ok def rdw2_opt_handler_backup_dle(errstack): global opt_mode opt_mode = 'backup-dle' return ade.ok def rdw2_opt_handler_check_dle(errstack): global opt_mode opt_mode = 'check-dle' return ade.ok def rdw2_opt_handler_backup_set(errstack): global opt_mode opt_mode = 'backup-set' return ade.ok def rdw2_opt_handler_check_set(errstack): global opt_mode opt_mode = 'check-set' return ade.ok def rdw2_opt_handler_list_misc(errstack): global opt_mode opt_mode = 'list-misc' return ade.ok def rdw2_opt_handler_list_backups(errstack): global opt_mode opt_mode = 'list-backups' return ade.ok def rdw2_opt_handler_list_medium_accesspoints(errstack): global opt_mode opt_mode = 'list-medium-accesspoints' return ade.ok def rdw2_opt_handler_list_pids(errstack): global opt_mode opt_mode = 'list-pids' return ade.ok def rdw2_opt_handler_list_medium_accesspoint(errstack): global opt_mode opt_mode = 'list-medium-accesspoint' return ade.ok def rdw2_opt_handler_no_update_cron(errstack): global opt_update_cron_flag opt_update_cron_flag = False return ade.ok def rdw2_opt_handler_db_file(errstack, db_file): global opt_db_file opt_db_file = db_file return ade.ok def rdw2_opt_handler_list_all(errstack): global opt_mode opt_mode = 'list-all' return ade.ok def rdw2_opt_handler_list_filters(errstack): global opt_mode opt_mode = 'list-filters' return ade.ok def rdw2_opt_handler_add_filter(errstack): global opt_mode opt_mode = 'add-filter' return ade.ok def rdw2_opt_handler_del_filter(errstack): global opt_mode opt_mode = 'del-filter' return ade.ok def rdw2_opt_handler_force(errstack): global opt_force_flag opt_force_flag = True return ade.ok ######################################################################## # # Functions that should be moved to ADE but are not mature enough yet # ######################################################################## def sql_lock_row(errstack, cursor, table_name, update_or_insert, primary_key_column_names, primary_key_column_values, set_column_names, set_column_values, entity_type, entity_name, errmsg_reformatter=None): # This function should only be called inside a transaction. ade.assert_inside_sql_transaction(errstack) if update_or_insert == 'update': # This is *identical* to the 'update' mode in sql_unlock_row(). set_column_names_string = ', '.join(set_column_names) set_column_values_string = ', '.join(['?' for x in range(0,len(set_column_values))]) where_condition_string = ' and '.join(['%s = ?' % (primary_key_column_name) for primary_key_column_name in primary_key_column_names]) sql_statement = 'UPDATE %s SET (%s) = (%s) WHERE %s;' % (table_name, set_column_names_string, set_column_values_string, where_condition_string) sql_value = tuple(set_column_values + primary_key_column_values) else: set_column_names_string = ', '.join(set_column_names) set_column_values_string = ', '.join(['?' for x in range(0,len(set_column_values))]) sql_statement = 'INSERT INTO %s (%s) VALUES (%s);' % (table_name, set_column_names_string, set_column_values_string) sql_value = tuple(set_column_values) ade.debug(errstack, 10, 'sql_lock_row: sql_statement = %s, sql_value = %s' % (sql_statement, sql_value)) #rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value, errmsg_reformatter) if rc != ade.ok: return rc # The UPDATE should update one row. (The INSERT should insert one row too # so we don't need to restrict this check to just the UPDATE clause above.) sql_statement = 'SELECT changes();' sql_value = () rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (count,) = select_results[0] if count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: no such %s (case #1)' % (entity_name, entity_type)) return ade.fail return ade.ok def sql_unlock_row(errstack, cursor, table_name, update_or_delete, primary_key_column_names, primary_key_column_values, set_column_names, set_column_values, entity_type, entity_name, errmsg_reformatter=None): # This function should only be called inside a transaction. ade.assert_inside_sql_transaction(errstack) if update_or_delete == 'update': # This is *identical* to the 'update' mode in sql_lock_row(). set_column_names_string = ', '.join(set_column_names) set_column_values_string = ', '.join(['?' for x in range(0,len(set_column_values))]) where_condition_string = ' and '.join(['%s = ?' % (primary_key_column_name) for primary_key_column_name in primary_key_column_names]) sql_statement = 'UPDATE %s SET (%s) = (%s) WHERE %s;' % (table_name, set_column_names_string, set_column_values_string, where_condition_string) sql_value = tuple(set_column_values + primary_key_column_values) else: where_condition_string = ' and '.join(['%s = ?' % (primary_key_column_name) for primary_key_column_name in primary_key_column_names]) sql_statement = 'DELETE FROM %s WHERE %s;' % (table_name, where_condition_string) sql_value = tuple(primary_key_column_values) ade.debug(errstack, 10, 'sql_unlock_row: sql_statement = %s, sql_value = %s' % (sql_statement, sql_value)) rc = ade.execute_sql_qm(errstack, cursor, sql_statement, sql_value, errmsg_reformatter) if rc != ade.ok: return rc # The UPDATE should update one row. (The DELETE should delete one row too # so we don't need to restrict this check to just the UPDATE clause above.) sql_statement = 'SELECT changes();' sql_value = () rc, select_results = ade.select_sql_qm(errstack, cursor, sql_statement, sql_value) if rc != ade.ok: return rc (count,) = select_results[0] if count != 1: ade.error(errstack, 'rdw2_err_misc', '%s: no such %s (case #2)' % (entity_name, entity_type)) return ade.fail return ade.ok ######################################################################## # # Functions that should be moved to ADE and *are* mature enough # ######################################################################## ######################################################################## # # Entry point # ######################################################################## ade.main(rdw2)