From 1c8ea280ac1badd78a263a719bf073e2f806bcfd Mon Sep 17 00:00:00 2001 From: kgashok Date: Wed, 15 Nov 2017 07:10:44 +0530 Subject: [PATCH 01/11] Update recipe-576917.py included a lambda version (<2.7?) and added a >3.5x version as well. --- .../recipe-576917.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/recipes/Python/576917_Functional_selection_sort/recipe-576917.py b/recipes/Python/576917_Functional_selection_sort/recipe-576917.py index f1630f62e..aec808a23 100644 --- a/recipes/Python/576917_Functional_selection_sort/recipe-576917.py +++ b/recipes/Python/576917_Functional_selection_sort/recipe-576917.py @@ -16,8 +16,33 @@ print test_list print selection_sort(test_list) + # Recursive version def select_sort_r(L): if not L: return [] # terminal case v, idx = min((v, i) for i, v in enumerate(L)) # select the smallest return [v] + select_sort_r(L[:idx] + L[idx+1:]) # recursively call on the remainder + + +# Lambda version +# Python > 2.7.10 and above +select_sort_l = ( + lambda L: [] if not L else ( + lambda idx, v: [v] + select_sort_l(L[:idx] + L[idx + 1:]))( + *min(enumerate(L), key=lambda t: t[1]) + ) +) + +# OP's version from the comments, works probably for Python < 2.7.10 +selsort2 = (lambda l: [] if not l else + (lambda (idx, v): [v] + selsort2(l[:idx] + l[idx + 1:])) + (min(enumerate(l), key=lambda(i, x): x))) + + +# OP's original test_list gets mutated by the functional version +test_list = [5, 1, 7, 0, -3, -10, 10, -6, 1, 0, 2, 4, -2, 6, 5, 8, 2] +expected = [-10, -6, -3, -2, 0, 0, 1, 1, 2, 2, 4, 5, 5, 6, 7, 8, 10] + +assert select_sort_r(test_list) == expected +assert select_sort_l(test_list) == expected +# assert selsort2(test_list) == expected From eb3910c872b400a25d803a709457011a3558f1ed Mon Sep 17 00:00:00 2001 From: elFua Date: Sun, 19 Nov 2017 17:09:09 +0100 Subject: [PATCH 02/11] find_replace_regex_colorized.py usage: find-replace-in-files-regex.py [-h] [--dir DIR] --search-regex SEARCH_REGEX --replace-regex REPLACE_REGEX [--glob GLOB] [--dry-run] [--create-backup] [--verbose] [--print-parent-folder] [--list-non-matching] DESCRIPTION: Find and replace recursively from the given folder using regular expressions optional arguments: -h, --help show this help message and exit --dir DIR, -d DIR folder to search in; by default current folder --search-regex SEARCH_REGEX, -s SEARCH_REGEX search regex --replace-regex REPLACE_REGEX, -r REPLACE_REGEX replacement regex --glob GLOB, -g GLOB glob pattern, i.e. *.html --dry-run, -dr don't replace anything just show what is going to be done --create-backup, -b Create backup files --verbose, -v Show files which don't match the search regex --print-parent-folder, -p Show the parent info for debug --list-non-matching, -n Supress colors USAGE: find-replace-in-files-regex.py -d [my_folder] -s -r -g [glob_pattern] --- ...recursive_file_regex_replacer_colorized.py | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 recipes/Python/recursive_file_regex_replacer_colorized.py diff --git a/recipes/Python/recursive_file_regex_replacer_colorized.py b/recipes/Python/recursive_file_regex_replacer_colorized.py new file mode 100644 index 000000000..4f6949311 --- /dev/null +++ b/recipes/Python/recursive_file_regex_replacer_colorized.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python + +""" + + + +Recursively find and replace text in files under a specific folder with preview of changed data in dry-run mode +============ + +Example Usage +--------------- + +**See what is going to change (dry run):** + +> flip all dates from 2017-12-31 to 31-12-2017 + + find_replace.py --dir project/myfolder --search-regex "\d{4}-\d{2}-\d{2}" --replace-regex "\3-\2-\1" --dry-run + +**Do actual replacement:** + + find_replace.py --dir project/myfolder --search-regex "\d{4}-\d{2}-\d{2}" --replace-regex "\3-\2-\1" + +**Do actual replacement and create backup files:** + + find_replace.py --dir project/myfolder --search-regex "\d{4}-\d{2}-\d{2}" --replace-regex "\3-\2-\1" --create-backup + +**Same action as previous command with short-hand syntax:** + + find_replace.py -d project/myfolder -s "\d{4}-\d{2}-\d{2}" -r "\3-\2-\1" -b + +Output of `find_replace.py -h`: + +usage: find-replace-in-files-regex.py [-h] [--dir DIR] --search-regex + SEARCH_REGEX --replace-regex + REPLACE_REGEX [--glob GLOB] [--dry-run] + [--create-backup] [--verbose] + [--print-parent-folder] + +USAGE: + find-replace-in-files-regex.py -d [my_folder] -s -r -g [glob_pattern] +""" + +from __future__ import print_function +import os +import fnmatch +import sys +import shutil +import re + +import argparse + + +class Colors: + Default = "\033[39m" + Black = "\033[30m" + Red = "\033[31m" + Green = "\033[32m" + Yellow = "\033[33m" + Blue = "\033[34m" + Magenta = "\033[35m" + Cyan = "\033[36m" + LightGray = "\033[37m" + DarkGray = "\033[90m" + LightRed = "\033[91m" + LightGreen = "\033[92m" + LightYellow = "\033[93m" + LightBlue = "\033[94m" + LightMagenta = "\033[95m" + LightCyan = "\033[96m" + White = "\033[97m" + NoColor = "\033[0m" + + +def find_replace(cfg): + + search_pattern = re.compile(cfg.search_regex) + + if cfg.dry_run: + print('THIS IS A DRY RUN -- NO FILES WILL BE CHANGED!') + + for path, dirs, files in os.walk(os.path.abspath(cfg.dir)): + for filename in fnmatch.filter(files, cfg.glob): + + if cfg.print_parent_folder: + pardir = os.path.normpath(os.path.join(path, '..')) + pardir = os.path.split(pardir)[-1] + print('[%s]' % pardir) + full_path = os.path.join(path, filename) + + # backup original file + if cfg.create_backup: + backup_path = full_path + '.bak' + + while os.path.exists(backup_path): + backup_path += '.bak' + print('DBG: creating backup', backup_path) + shutil.copyfile(full_path, backup_path) + + if os.path.islink(full_path): + print("{}File {} is a symlink. Skipping{}".format(Colors.Red, full_path, Colors.NoColor)) + continue + + with open(full_path) as f: + old_text = f.read() + + all_matches = search_pattern.findall(old_text) + + if all_matches: + + print('{}Found {} match(es) in file {}{}'.format(Colors.LightMagenta, len(all_matches), filename, Colors.NoColor)) + + new_text = search_pattern.sub(cfg.replace_regex, old_text) + + if not cfg.dry_run: + with open(full_path, "w") as f: + print('DBG: replacing in file', full_path) + f.write(new_text) + # else: + # for idx, matches in enumerate(all_matches): + # print("Match #{}: {}".format(idx, matches)) + + if cfg.verbose or cfg.dry_run: + colorized_old = search_pattern.sub(Colors.LightBlue + r"\g<0>" + Colors.NoColor, old_text) + colorized_old = '\n'.join(['\t' + line.strip() for line in colorized_old.split('\n') if Colors.LightBlue in line]) + + colorized = search_pattern.sub(Colors.Green + cfg.replace_regex + Colors.NoColor, old_text) + colorized = '\n'.join(['\t' + line.strip() for line in colorized.split('\n') if Colors.Green in line]) + print("{}BEFORE:{}\n{}".format(Colors.White, Colors.NoColor, colorized_old)) + print("{}AFTER :{}\n{}".format(Colors.Yellow, Colors.NoColor, colorized)) + + elif cfg.list_non_matching: + print('File {} does not contain search regex "{}"'.format(filename, cfg.search_regex)) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='''DESCRIPTION: + Find and replace recursively from the given folder using regular expressions''', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog='''USAGE: + {0} -d [my_folder] -s -r -g [glob_pattern] + + '''.format(os.path.basename(sys.argv[0]))) + + parser.add_argument('--dir', '-d', + help='folder to search in; by default current folder', + default='.') + + parser.add_argument('--search-regex', '-s', + help='search regex', + required=True) + + parser.add_argument('--replace-regex', '-r', + help='replacement regex', + required=True) + + parser.add_argument('--glob', '-g', + help='glob pattern, i.e. *.html', + default="*.*") + + parser.add_argument('--dry-run', '-dr', + action='store_true', + help="don't replace anything just show what is going to be done", + default=False) + + parser.add_argument('--create-backup', '-b', + action='store_true', + help='Create backup files', + default=False) + + parser.add_argument('--verbose', '-v', + action='store_true', + help="Show files which don't match the search regex", + default=False) + + parser.add_argument('--print-parent-folder', '-p', + action='store_true', + help="Show the parent info for debug", + default=False) + + parser.add_argument('--list-non-matching', '-n', + action='store_true', + help="Supress colors", + default=False) + + config = parser.parse_args(sys.argv[1:]) + + find_replace(config) + From d82d03e737eb8e2a978eac4ff6c9c9e19c2efc71 Mon Sep 17 00:00:00 2001 From: Pavel Shishpor Date: Fri, 24 Nov 2017 15:01:17 +0100 Subject: [PATCH 03/11] Explicitly import rlcompleter as it may not be visible when imported in .pdbrc. The issue was observed on Python 2.7.13. --- .../Python/498182_sane_tab_completion_in_pdb/recipe-498182.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/Python/498182_sane_tab_completion_in_pdb/recipe-498182.py b/recipes/Python/498182_sane_tab_completion_in_pdb/recipe-498182.py index 3b42c9d14..a2427d233 100644 --- a/recipes/Python/498182_sane_tab_completion_in_pdb/recipe-498182.py +++ b/recipes/Python/498182_sane_tab_completion_in_pdb/recipe-498182.py @@ -22,6 +22,7 @@ def complete(self, text, state): """ # keep a completer class, and make sure that it uses the current local scope if not hasattr(self, 'completer'): + import rlcompleter self.completer = rlcompleter.Completer(self.curframe.f_locals) else: self.completer.namespace = self.curframe.f_locals From 8e2528c312791c2c56c8bca365669e06ab698a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Niklas?= Date: Wed, 29 Nov 2017 08:00:27 +0100 Subject: [PATCH 04/11] omitting temporary tables and similar improvements --- .../recipe-576534.py | 365 ++++++++++----- .../recipe-576557.py | 432 ++++++++++++++---- .../recipe-576621.py | 397 +++++++++++----- 3 files changed, 894 insertions(+), 300 deletions(-) diff --git a/recipes/Python/576534_Dump_Oracle_db_schema_to_text/recipe-576534.py b/recipes/Python/576534_Dump_Oracle_db_schema_to_text/recipe-576534.py index e89501fb6..e6683673b 100644 --- a/recipes/Python/576534_Dump_Oracle_db_schema_to_text/recipe-576534.py +++ b/recipes/Python/576534_Dump_Oracle_db_schema_to_text/recipe-576534.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -__version__ = '$Id: schema_ora.py 1754 2014-02-14 08:57:52Z mn $' +__version__ = '$Id: schema_ora.py 3307 2017-11-29 06:18:57Z mn $' # export Oracle schema to text # usable to compare databases that should be the same @@ -8,29 +8,33 @@ # Oracle schema info: # http://www.eveandersson.com/writing/data-model-reverse-engineering # +# http://code.activestate.com/recipes/576534-dump-oracle-db-schema-to-text/ # # author: Michal Niklas, Adam KopciƄski-Galik OPTIONS = """[OPTIONS] Options: ---tables_only raport only tables ---separate-files save tables, views etc in separate files - in db_schema directory, - tables are defined as CREATE TABLE statement ---sorted-info for CREATE TABLE add comment with columns - sorted by name ---zip with --separate-files enabled zip all created files +--add-ver-info add version information --date-dir with --separate-files add date and time to db_schema_[date]_[time] directory with -o[=file_name] save file_name in db_schema_[date]_[time] directory --force-dir with --separate-files do not check if db_schema directory exists +--lf-only use Unix style end of lines +--no-temp-tables do not show info about tables with 'tmp' or 'temp' in name -o[=file_name] send results to file instead of stdout +--separate-files save tables, views etc in separate files + in db_schema directory, + tables are defined as CREATE TABLE statement +--sorted-info for CREATE TABLE add comment with columns + sorted by name +--tables-only raport only tables --version show version ---add-ver-info add version information +--verbose show more info in output (SQL queries) --ver-info-sql[=SELECT ... FROM ... ORDER BY ...] extend version information by results of this query +--zip with --separate-files enabled zip all created files """ USAGE = 'usage:\n\tschema_ora.py tnsentry username passwd %s' % OPTIONS @@ -41,22 +45,25 @@ """ % OPTIONS import codecs -import sys import os -import zipfile import os.path -import time import re +import sys +import time +import traceback +import zipfile USE_JYTHON = 0 -TABLES_ONLY = 0 +TABLES_ONLY = '--tables_only' in sys.argv or '--tables-only' in sys.argv +LF_ONLY = '--lf_only' in sys.argv or '--lf-only' in sys.argv + SCHEMA_DIR = 'db_schema' if '--date-dir' in sys.argv: SCHEMA_DIR += time.strftime("_%y%m%d_%H%M%S", time.localtime()) -TABLES_INFO_DIR = SCHEMA_DIR + '/tables' -VIEWS_INFO_DIR = SCHEMA_DIR + '/views' +TABLES_INFO_DIR = SCHEMA_DIR + '/tables' +VIEWS_INFO_DIR = SCHEMA_DIR + '/views' SEQUENCES_INFO_DIR = SCHEMA_DIR + '/sequences' FUNCTIONS_INFO_DIR = SCHEMA_DIR + '/functions' PROCEDURES_INFO_DIR = SCHEMA_DIR + '/procedures' @@ -65,6 +72,7 @@ CREATED_FILES = [] +VERBOSE = '--verbose' in sys.argv try: from com.ziclix.python.sql import zxJDBC @@ -78,30 +86,40 @@ OUT_FILE_ENCODING = 'UTF8' - -TABLE_AND_VIEWS_NAMES_SQL = """SELECT DISTINCT table_name -FROM user_tab_columns -WHERE INSTR(table_name, 'X_') <> 1 +FILTER_TEMP = """ +INSTR(table_name, 'X_') <> 1 AND INSTR(table_name, '$') = 0 -ORDER BY table_name """ +FILTER_TEMP_SEQ = "" + +if '--no-temp-tables' in sys.argv: + # Oracle uses UPPER letters in table names + FILTER_TEMP += """AND INSTR(table_name, 'TMP') = 0 +AND INSTR(table_name, 'TEMP') = 0 +""" + FILTER_TEMP_SEQ = " WHERE INSTR(sequence_name, 'TMP') = 0 AND INSTR(sequence_name, 'TEMP') = 0" + +UC_FILTER_TEMP = FILTER_TEMP.replace('table_name', 'uc.table_name') + TABLE_NAMES_SQL = """SELECT DISTINCT table_name FROM user_tables -WHERE INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') = 0 +WHERE +%s AND NOT table_name IN (SELECT view_name FROM user_views) AND NOT table_name IN (SELECT mview_name FROM user_mviews) ORDER BY table_name -""" +""" % (FILTER_TEMP) TABLE_COLUMNS_SQL = """SELECT table_name, column_name FROM user_tab_columns -WHERE INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') = 0 +WHERE +%s +AND NOT table_name IN (SELECT view_name FROM user_views) +AND NOT table_name IN (SELECT mview_name FROM user_mviews) ORDER BY table_name, column_name -""" +""" % (FILTER_TEMP) TABLE_INFO_SQL = """SELECT table_name, column_name, data_type, nullable, @@ -109,53 +127,78 @@ decode(data_type, 'DATE', '11', 'NUMBER', data_precision || ',' || data_scale, + 'VARCHAR2', char_length || char_used, data_length) data_length FROM user_tab_columns -WHERE INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') = 0 +WHERE +%s +AND NOT table_name IN (SELECT view_name FROM user_views) +AND NOT table_name IN (SELECT mview_name FROM user_mviews) ORDER BY table_name, column_name -""" +""" % (FILTER_TEMP) + + +SQL_TYPES = """SELECT COUNT(*), data_type, +decode(data_type, + 'NUMBER', data_precision || ',' || data_scale, + 'VARCHAR2', char_used, + 'CHAR', char_used, + data_length) +FROM user_tab_columns +WHERE +%s +AND NOT table_name IN (SELECT view_name FROM user_views) +AND NOT table_name IN (SELECT mview_name FROM user_mviews) +GROUP BY data_type, +decode(data_type, + 'NUMBER', data_precision || ',' || data_scale, + 'VARCHAR2', char_used, + 'CHAR', char_used, + data_length) +ORDER BY 1 DESC, 2, 3 +""" % (FILTER_TEMP) + PRIMARY_KEYS_INFO_SQL = """SELECT uc.table_name, ucc.column_name FROM user_constraints uc, user_cons_columns ucc -WHERE uc.constraint_name = ucc.constraint_name +WHERE +%s +AND uc.constraint_name = ucc.constraint_name AND uc.constraint_type = 'P' -AND INSTR(uc.table_name, 'X_') <> 1 -AND INSTR(uc.table_name, '$') = 0 ORDER BY uc.table_name, ucc.position -""" +""" % (UC_FILTER_TEMP) INDEXES_INFO_SQL = """SELECT ui.table_name, ui.index_name, ui.uniqueness FROM user_indexes ui -WHERE INSTR(ui.table_name, 'X_') <> 1 -AND INSTR(ui.table_name, '$') = 0 +WHERE +%s ORDER BY ui.table_name, ui.index_name -""" +""" % (FILTER_TEMP) INDEXES_COLUMNS_INFO_SQL = """SELECT table_name, column_name, index_name, column_position, descend FROM user_ind_columns -WHERE INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') = 0 +WHERE +%s ORDER BY table_name, index_name, column_position -""" +""" % (FILTER_TEMP) COMPOSITE_INDEXES_COLUMNS_INFO_SQL = """SELECT table_name, column_name, index_name, column_position FROM user_ind_columns -WHERE index_name in (select distinct index_name from USER_IND_COLUMNS where column_position > 1) -AND INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') = 0 +WHERE +%s +AND index_name in (select distinct index_name from USER_IND_COLUMNS where column_position > 1) ORDER BY table_name, index_name, column_position -""" +""" % (FILTER_TEMP) FUNCTION_INDEXES_INFO_SQL = """SELECT table_name, column_expression, index_name, column_position FROM user_ind_expressions -WHERE INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') = 0 +WHERE +%s ORDER BY table_name, index_name, column_position -""" +""" % (FILTER_TEMP) FOREIGN_KEYS_INFO_SQL = """SELECT uc.table_name, ucc.column_name, ucc.position @@ -166,29 +209,36 @@ ,user_constraints fc ,user_constraints uc ,user_ind_columns uic -WHERE uc.constraint_type = 'R' -AND uc.constraint_name = ucc.constraint_name -AND fc.constraint_name = uc.r_constraint_name +WHERE +%s +AND uc.constraint_type = 'R' +AND uc.constraint_name = ucc.constraint_name +AND fc.constraint_name = uc.r_constraint_name AND uic.index_name=fc.constraint_name ORDER BY uc.table_name, ucc.position, uic.column_position -""" +""" % (UC_FILTER_TEMP) DEFAULTS_INFO_SQL = """SELECT table_name, column_name, data_default -FROM user_tab_columns -WHERE default_length IS NOT NULL -AND INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') = 0 +FROM user_tab_columns +WHERE +%s +AND default_length IS NOT NULL ORDER BY table_name, column_name -""" +""" % (FILTER_TEMP) -SEQUENCES_INFO_SQL = """SELECT sequence_name FROM user_sequences""" +SEQUENCES_INFO_SQL = """SELECT sequence_name +FROM user_sequences %s +ORDER BY sequence_name +""" % FILTER_TEMP_SEQ TSEQUENCES_INFO_SQL = """SELECT sequence_name, min_value, max_value, increment_by, last_number, cache_size, cycle_flag, order_flag FROM user_sequences -""" +%s +ORDER BY sequence_name +""" % FILTER_TEMP_SEQ VIEWS_INFO_SQL = """SELECT view_name, text @@ -204,20 +254,27 @@ TRIGGERS_INFO_SQL = """SELECT trigger_name, trigger_type, triggering_event, table_name, trim(chr(13) from trim(chr(10) from description)), trigger_body FROM user_triggers -WHERE INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') = 0 +WHERE +%s ORDER BY table_name, trigger_name +""" % (FILTER_TEMP) + + +PROCEDURE_AND_FUNCTION_NAME_SQL = """SELECT object_name +FROM user_procedures +WHERE procedure_name IS NULL AND LOWER(object_type)=LOWER('%s') +ORDER BY object_name """ TTABLE_NAMES_SQL = """SELECT DISTINCT table_name FROM user_tab_columns -WHERE INSTR(table_name, 'X_') <> 1 -AND INSTR(table_name, '$') < 1 +WHERE +%s AND NOT table_name IN (SELECT view_name FROM user_views) AND NOT table_name IN (SELECT mview_name FROM user_mviews) ORDER BY table_name -""" +""" % (FILTER_TEMP) TTABLE_COLUMNS = """SELECT column_name, data_type, nullable, @@ -227,7 +284,8 @@ 'NUMBER', data_precision || ',' || data_scale, data_length) data_length, data_default, - char_length + char_length, + char_used FROM user_tab_columns WHERE table_name='%s' """ @@ -240,6 +298,7 @@ WHERE uc.constraint_name = ucc.constraint_name AND uc.constraint_type = 'P' AND uc.table_name='%s' +ORDER BY ucc.position """ @@ -277,8 +336,19 @@ """ +TTYPES_SQL = """SELECT type_name, line, text FROM user_types +LEFT JOIN user_source ON user_source.name=user_types.type_name WHERE user_source.type='TYPE' +ORDER BY type_name, line""" + + DB_VERSION_SQL = """SELECT * FROM v$version WHERE banner like 'Oracle%'""" +FEATURE_USAGE_SQL = """SELECT NAME, VERSION, detected_usages, +first_usage_date, last_usage_date +FROM dba_feature_usage_statistics +WHERE detected_usages > 0 +ORDER BY 1, 2""" + _CONN = None @@ -323,8 +393,8 @@ def init_db_conn(connect_string, username, passwd): print('--%s' % (dbinfo)) _CONN = cx_Oracle.connect(username, passwd, connect_string) except: - ex = sys.exc_info() - serr = 'Exception: %s: %s\n%s' % (ex[0], ex[1], dbinfo) + serr = traceback.format_exc() + serr = 'Exception while opening: %s\n%s' % (dbinfo, serr) print_err(serr) return None return _CONN @@ -339,6 +409,8 @@ def output_str(fout, line): """outputs line to fout trying various encodings in case of encoding errors""" if fout: try: + if LF_ONLY: + line = line.replace(chr(13), '') fout.write(line) except (UnicodeDecodeError, UnicodeEncodeError): try: @@ -357,13 +429,17 @@ def output_str(fout, line): if not ok: fout.write('!!! line cannot be encoded !!!\n') fout.write(repr(line)) - fout.write('\n') + if LF_ONLY: + # just LF, while we want to compare results on both Linux and Windows + # CRLF = 0d0a + fout.write(chr(10)) + else: + fout.write('\n') fout.flush() def output_line(line, fout=None): """outputs line""" - line = line.rstrip() output_str(fout, line) output_str(sys.stdout, line) @@ -371,15 +447,24 @@ def output_line(line, fout=None): def print_err(serr): """println on stderr""" sys.stderr.write('%s\n' % (serr)) + output_line('\nERROR! ERROR!\n%s\n' % (serr)) def select_qry(querystr): """executes SQL SELECT query""" - cur = db_conn().cursor() - cur.execute(querystr) - results = cur.fetchall() - cur.close() - return results + try: + qstr = querystr.encode('UTF8') + cur = db_conn().cursor() + if VERBOSE: + output_line('\n--\n%s\n--' % (qstr)) + cur.execute(qstr) + results = cur.fetchall() + cur.close() + return results + except: + serr = traceback.format_exc() + print_err('cos nie tak z zapytaniem:\n%s\n%s' % (qstr, serr)) + raise def run_qry(querystr): @@ -403,16 +488,29 @@ def fld2str(fld_v): return fld_v +def print_start_info(title, fout=None): + output_line('\n\n', fout) + output_line('--- %s (START) ---' % title, fout) + + +def print_stop_info(title, fout=None): + output_line('--- %s (END) ---' % title, fout) + output_line('\n\n', fout) + + def show_qry(title, querystr, fld_join='\t', row_separator=None, fout=None): """shows SQL query results""" + print_start_info(title, fout) rs = select_qry(querystr) if rs: - output_line('\n\n--- %s ---' % (title), fout) for row in rs: line = fld_join.join([fld2str(s) for s in row]) output_line(line, fout) if row_separator: output_line(row_separator, fout) + else: + output_line(' -- NO DATA --', fout) + print_stop_info(title, fout) def init_session(): @@ -420,7 +518,7 @@ def init_session(): run_qry("ALTER SESSION SET nls_numeric_characters = '.,'") -def get_type_length(data_type, data_length, char_length): +def get_type_length(data_type, data_length, char_length, char_used): """get string with length of field""" if data_type == 'NUMBER': if data_length == ',': @@ -431,7 +529,10 @@ def get_type_length(data_type, data_length, char_length): if data_type == 'RAW': return ' (%s)' % (data_length) if data_type in ('CHAR', 'VARCHAR2', 'NCHAR', 'NVARCHAR2'): - return ' (%.0f)' % (char_length) + char_used_str = 'BYTE' + if char_used.lower().startswith('c'): + char_used_str = 'CHAR' + return ' (%.0f %s)' % (char_length, char_used_str) return '' @@ -444,8 +545,9 @@ def table_info_row(row): data_length = row[4] data_default = row[5] char_length = row[6] + char_used = row[7] default_str = nullable_str = '' - data_length_str = get_type_length(data_type, data_length, char_length) + data_length_str = get_type_length(data_type, data_length, char_length, char_used) if int(hasdef) == 1: default_str = ' DEFAULT %s' % (data_default) if nullable == 'N': @@ -595,8 +697,12 @@ def create_create_table_ddl(table, sorted_in_comment): sc = '\n---------- order by column name ----------\n-- ' + sc + '\n---------- order by column name ----------' ct = ct + sc indices_str = get_table_indices(table, pk_columns) + if indices_str: + ct += '\n\n' + indices_str triggers_str = get_table_triggers(table) - return '%s\n\n%s\n\n%s' % (ct, indices_str, triggers_str) + if triggers_str: + ct += '\n\n' + triggers_str + return ct def save_table_definition(table, sorted_in_comment): @@ -611,7 +717,7 @@ def save_table_definition(table, sorted_in_comment): def show_tables(): """shows info tables""" - show_qry('tables', TABLE_AND_VIEWS_NAMES_SQL) + show_qry('tables', TABLE_NAMES_SQL) show_qry('table columns', TABLE_COLUMNS_SQL) show_qry('columns', TABLE_INFO_SQL) @@ -709,6 +815,16 @@ def show_views(separate_files): cur.close() +def show_types_stat(): + show_qry('column type count', SQL_TYPES) + + +def show_procedures_and_functions(): + """shows SQL triggers""" + show_qry('function names', PROCEDURE_AND_FUNCTION_NAME_SQL % ('function')) + show_qry('procedure names', PROCEDURE_AND_FUNCTION_NAME_SQL % ('procedure')) + + def show_normal_procedures(separate_files, title, out_dir=None): """shows valid SQL procedures and functions""" return show_procedures("SELECT object_name FROM user_procedures WHERE procedure_name IS NULL AND lower(object_type) = lower('%s') ORDER BY 1", separate_files, title, out_dir) @@ -721,7 +837,7 @@ def show_invalid_procedures(separate_files, title, out_dir=None): def show_procedures(sql, separate_files, title, out_dir=None): """shows SQL procedures and functions""" - output_line('\n\n --- %ss ---' % (title)) + print_start_info(title + 's') fout = None cur = db_conn().cursor() cur.execute(sql % (title)) @@ -738,19 +854,21 @@ def show_procedures(sql, separate_files, title, out_dir=None): cur2.execute("SELECT text FROM user_source where name = '%s' ORDER BY line" % funname) lines = cur2.fetchall() for line in lines: - output_line(line[0], fout) + output_line(line[0].rstrip(), fout) if lines: - output_line('\n/\n', fout) + output_line('\n/', fout) cur2.close() output_line('\n\n -- <<< %s %s <<< --' % (title, funname)) if fout: fout.close() cur.close() + print_stop_info(title + 's') def show_packages(separate_files): """shows SQL packages""" - output_line('\n\n --- packages ---') + title = 'packages' + print_start_info(title) fout = None cur1 = db_conn().cursor() cur1.execute("SELECT object_name FROM user_objects WHERE object_type='PACKAGE' ORDER BY 1") @@ -766,9 +884,9 @@ def show_packages(separate_files): cur2.execute("SELECT text FROM sys.user_source where name = '%s' AND type='PACKAGE' ORDER BY line" % funname) lines = cur2.fetchall() for line in lines: - output_line(line[0], fout) + output_line(line[0].rstrip(), fout) if(lines): - output_line('\n/\n', fout) + output_line('\n/', fout) output_line('\n') cur2.close() cur3 = db_conn().cursor() @@ -777,14 +895,15 @@ def show_packages(separate_files): if(lines): output_line('\nCREATE OR REPLACE ', fout) for line in lines: - output_line(line[0], fout) + output_line(line[0].rstrip(), fout) if(lines): - output_line('\n/\n', fout) + output_line('\n/', fout) cur3.close() if fout: fout.close() output_line('\n\n -- <<< package %s <<< --' % (funname)) cur1.close() + print_stop_info(title) def show_triggers(): @@ -792,6 +911,11 @@ def show_triggers(): show_qry('triggers', TRIGGERS_INFO_SQL, '\n', '\n-- end trigger --\n') +def show_types(): + """shows SQL types""" + show_qry('types', TTYPES_SQL) + + def save_files_in_zip(): """saves created files in zip file""" if not CREATED_FILES: @@ -815,12 +939,14 @@ def add_ver_info(separate_files, connect_string, username): if separate_files: ver_file = os.path.join(SCHEMA_DIR, 'version.txt') f = open_file_write(ver_file) - output_line('>>> ver info') + title = 'info' + print_start_info(title) output_line('date: %s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())), f) output_line('connect string: %s' % (connect_string), f) output_line('user: %s' % (username), f) script_ver = __version__[5:-2] output_line('created by: %s' % (script_ver), f) + print_stop_info(title) show_qry('DB version', DB_VERSION_SQL, fout=f) show_qry('DB name', 'SELECT ora_database_name FROM dual', fout=f) sel_info_option = '--ver-info-sql' @@ -830,11 +956,11 @@ def add_ver_info(separate_files, connect_string, username): try: show_qry(sel, sel, fout=f) except: - ex = sys.exc_info() - serr = '\nSQL: %s\nException: %s: %s\n' % (sel, ex[0], ex[1]) + ex_info = traceback.format_exc() + serr = '\nSQL: %s\nException: %s\n' % (sel, ex_info) print_err(serr) break - output_line('<<< ver info') + show_qry('DB features used', FEATURE_USAGE_SQL, fout=f) if f: f.close() @@ -848,22 +974,33 @@ def show_additional_info(separate_files): show_sequences(separate_files) show_views(separate_files) show_triggers() + show_procedures_and_functions() show_normal_procedures(separate_files, 'function', FUNCTIONS_INFO_DIR) - show_invalid_procedures(separate_files, 'function', FUNCTIONS_INFO_DIR + INVALID) + show_invalid_procedures(separate_files, 'invalid function', FUNCTIONS_INFO_DIR + INVALID) show_normal_procedures(separate_files, 'procedure', PROCEDURES_INFO_DIR) - show_invalid_procedures(separate_files, 'procedure', PROCEDURES_INFO_DIR + INVALID) + show_invalid_procedures(separate_files, 'invalid procedure', PROCEDURES_INFO_DIR + INVALID) show_packages(separate_files) + show_types() + show_types_stat() + + +def rmdir_ex(dirname): + try: + os.rmdir(dirname) + except: + pass def clean_up(): """removes created directories after zipping files""" for dn in CREATED_DIRS: - os.rmdir(dn) - os.rmdir(SCHEMA_DIR) + rmdir_ex(dn) + rmdir_ex(SCHEMA_DIR) def dump_db_info(separate_files, out_f, stdout): """saves information about database schema in file/files""" + t0 = time.time() test = '--test' in sys.argv if test or separate_files: for dn in (TABLES_INFO_DIR, VIEWS_INFO_DIR, SEQUENCES_INFO_DIR, FUNCTIONS_INFO_DIR, PROCEDURES_INFO_DIR, PACKAGES_INFO_DIR): @@ -880,7 +1017,10 @@ def dump_db_info(separate_files, out_f, stdout): show_tables() if not TABLES_ONLY and not test: show_additional_info(separate_files) - output_line('\n\n--- the end ---') + t2 = time.time() + td = t2 - t0 + output_line('\n\nexecution time: %d min %d sec' % (divmod(td, 60))) + output_line('\n\n-- the end --') if out_f: out_f.close() sys.stdout = stdout @@ -899,25 +1039,37 @@ def get_option_value(prefix): return result +def mk_schema_dir(): + try: + if not os.path.exists(SCHEMA_DIR): + os.mkdir(SCHEMA_DIR) + except: + pass + + def main(): """main function""" - conn_args = [s for s in sys.argv[1:] if not s.startswith('-')] - if len(conn_args) != 3: + if '--version' in sys.argv: + print(__version__) + return + db_conn_args = [s for s in sys.argv[1:] if not s.startswith('-')] + if len(db_conn_args) != 3 or '--help' in sys.argv: print(USAGE) - return 0 - connect_string, username, passwd = conn_args + return + connect_string, username, passwd = db_conn_args separate_files = '--separate-files' in sys.argv if separate_files: if os.path.exists(SCHEMA_DIR): - if not '--force-dir' in sys.argv: + if '--force-dir' not in sys.argv: print_err('Output directory "%s" already exists,\nuse --force-dir or --date-dir option!' % (SCHEMA_DIR)) return 0 + mk_schema_dir() stdout = sys.stdout out_f = None out_fn = get_option_value('-o') if out_fn: if '--date-dir' in sys.argv: - os.mkdir(SCHEMA_DIR) + mk_schema_dir() out_fn = os.path.join(SCHEMA_DIR, out_fn) out_f = open(out_fn, 'w') sys.stdout = out_f @@ -932,12 +1084,5 @@ def main(): dump_db_info(separate_files, out_f, stdout) -if '--version' in sys.argv: - print(__version__) -elif __name__ == '__main__': - if '--tables_only' in sys.argv: - TABLES_ONLY = 1 - if len(sys.argv) < 4: - print(USAGE) - else: - main() +if __name__ == '__main__': + main() diff --git a/recipes/Python/576557_Dump_PostgreSQL_db/recipe-576557.py b/recipes/Python/576557_Dump_PostgreSQL_db/recipe-576557.py index 2b7644a76..9c0c0e041 100644 --- a/recipes/Python/576557_Dump_PostgreSQL_db/recipe-576557.py +++ b/recipes/Python/576557_Dump_PostgreSQL_db/recipe-576557.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -__version__ = '$Id: schema_pg.py 1754 2014-02-14 08:57:52Z mn $' +__version__ = '$Id: schema_pg.py 3307 2017-11-29 06:18:57Z mn $' # export PostgreSQL schema to text # usable to compare databases that should be the same @@ -8,24 +8,37 @@ # PostgreSQL schema info: # http://www.alberton.info/postgresql_meta_info.html # +# https://code.activestate.com/recipes/576557-dump-postgresql-db-schema-to-text/ # # author: Michal Niklas -USAGE = 'usage:\n\tschema_pg.py connect_string\n\t\tconnect string:\n\t\t\thost:[port:]database:user:password\n\t\tor for ODBC:\n\t\t\tdatabase/user/passwd\n\t\tor (pyodbc)\n\t\t\tDriver={PostgreSQL};Server=IP address;Port=5432;Database=myDataBase;Uid=myUsername;Pwd=myPassword;' +USAGE = """usage: + schema_pg.py connect_string + connect string: + host:[port:]database:user:password + or for ODBC: + database/user/passwd + or (pyodbc) + Driver={PostgreSQL};Server=IP address;Port=5432;Database=myDataBase;Uid=myUsername;Pwd=myPassword; +""" -import sys import array import exceptions +import sys +import time +import traceback USE_JYTHON = 0 -TABLES_ONLY = 0 +TABLES_ONLY = '--tables_only' in sys.argv + +OUTPUT_FILE = sys.stdout try: from com.ziclix.python.sql import zxJDBC USE_JYTHON = 1 USAGE = """usage: -\tschema_inf.py jdbcurl user passwd +\tschema_py.py jdbcurl user passwd example: \tjython schema_py.py jdbc:postgresql://isof-test64:5434/gryfcard_mrb?stringtype=unspecified user passwd > db.schema 2> db.err """ @@ -38,10 +51,14 @@ OUT_FILE_ENCODING = 'UTF8' +DB_VERSION_SQL = """SELECT version()""" + TABLE_NAMES_SQL = """SELECT DISTINCT table_name FROM information_schema.columns WHERE table_schema='public' AND position('_' in table_name) <> 1 +AND position('pg_' in table_name) <> 1 +AND NOT table_name IN (SELECT viewname FROM pg_views) ORDER BY 1""" @@ -49,25 +66,69 @@ FROM information_schema.columns WHERE table_schema='public' AND position('_' in table_name) <> 1 +AND position('pg_' in table_name) <> 1 +AND position('_' in column_name) <> 1 +AND NOT table_name IN (SELECT viewname FROM pg_views) ORDER BY 1, 2""" +TABLE_COLUMN_NAME_BY_IDX = """SELECT column_name +FROM information_schema.columns +WHERE table_name = '%s' +AND ordinal_position = %s""" + -TABLE_INFO_SQL = """SELECT table_name, column_name, ordinal_position, data_type, is_nullable, character_maximum_length, numeric_precision +TABLE_INFO_SQL = """SELECT table_name, column_name, +case + when data_type='numeric' and numeric_precision is not null then data_type || '(' || numeric_precision || ',' || numeric_scale || ')' + when character_maximum_length is not null then data_type || '(' || character_maximum_length || ')' + else data_type end as data_type, +is_nullable FROM information_schema.columns WHERE table_schema='public' AND position('_' in table_name) <> 1 +AND position('pg_' in table_name) <> 1 +AND NOT table_name IN (SELECT viewname FROM pg_views) +AND position('_' in column_name) <> 1 ORDER BY 1, 2 """ -PRIMARY_KEYS_INFO_SQL = """SELECT t.relname AS table_name, array_to_string(c.conkey, ' ') AS constraint_key +SQL_TYPES = """SELECT COUNT(*), case when data_type='numeric' and numeric_precision is not null then data_type || '(' || numeric_precision || ',' || numeric_scale || ')' else data_type end +FROM information_schema.columns +WHERE table_schema='public' +AND position('_' in table_name) <> 1 +AND position('pg_' in table_name) <> 1 +AND NOT table_name IN (SELECT viewname FROM pg_views) +AND position('_' in column_name) <> 1 +GROUP BY 2 +ORDER BY 1 DESC, 2 +""" + + +xxPRIMARY_KEYS_INFO_SQL = """SELECT t.relname AS table_name, array_to_string(c.conkey, ' ') AS constraint_key FROM pg_constraint c LEFT JOIN pg_class t ON c.conrelid = t.oid WHERE c.contype = 'p' -AND position('_' in t.relname ) <> 1 +AND position('_' in t.relname) <> 1 +AND position('pg_' in t.relname) <> 1 ORDER BY table_name; """ +PRIMARY_KEYS_INFO_SQL = """select tc.table_name, kc.column_name +from + information_schema.table_constraints tc, + information_schema.key_column_usage kc +where + tc.constraint_type = 'PRIMARY KEY' + and tc.table_schema = 'public' + and position('_' in tc.table_name) <> 1 + and position('pg_' in tc.table_name) <> 1 + and kc.table_name = tc.table_name and kc.table_schema = tc.table_schema + and kc.constraint_name = tc.constraint_name +order by 1, 2; +""" + + INDEXES_INFO_SQL = """SELECT relname, indisunique FROM pg_class, pg_index WHERE pg_class.oid = pg_index.indexrelid @@ -107,14 +168,41 @@ AND a.attnum = %s """ - -FOREIGN_KEYS_INFO_SQL = """SELECT t.relname AS table_name, t2.relname AS references_table -FROM pg_constraint c -LEFT JOIN pg_class t ON c.conrelid = t.oid -LEFT JOIN pg_class t2 ON c.confrelid = t2.oid -WHERE c.contype = 'f' -AND position('_' in t.relname ) <> 1 -ORDER BY table_name, references_table +# http://stackoverflow.com/questions/1567051/introspect-postgresql-8-3-to-find-foreign-keys +FOREIGN_KEYS_INFO_SQL = """SELECT "table", array_agg(columns)::varchar, "foreign table", array_agg("foreign columns")::varchar, conname, 'ALTER TABLE ONLY ' || "table" || ' ADD CONSTRAINT ' || conname || ' FOREIGN KEY (' || trim(both '{}' from array_agg(columns)::varchar) || ') REFERENCES ' || "foreign table" || '(' || trim(both '{}' from array_agg("foreign columns")::varchar) || ');' +FROM (SELECT conrelid::regclass AS "table", + a.attname as columns, + confrelid::regclass as "foreign table", + af.attname as "foreign columns", + conname + FROM pg_attribute AS af, + pg_attribute AS a, + (SELECT conrelid, + confrelid, + conkey[i] AS conkey, + confkey[i] as confkey, + conname + FROM (SELECT conrelid, + confrelid, + conkey, + confkey, + generate_series(1, array_upper(conkey, 1)) AS i, + conname + FROM pg_constraint + WHERE contype = 'f' + ) AS ss + ) AS ss2 + WHERE af.attnum = confkey + AND af.attrelid = confrelid + AND a.attnum = conkey + AND a.attrelid = conrelid + AND position('_' in conrelid::regclass::varchar) <> 1 + AND position('pg_' in conrelid::regclass::varchar) <> 1 +) AS ss3 +GROUP BY "table", + "foreign table", + conname +ORDER BY 1, 2, 3, 4; """ @@ -122,12 +210,26 @@ FROM information_schema.columns WHERE column_default IS NOT NULL AND position('_' in table_name) <> 1 +AND position('pg_' in table_name) <> 1 +AND position('_' in column_name) <> 1 ORDER BY table_name, column_name""" -VIEWS_INFO_SQL = """SELECT viewname, definition +VIEW_NAMES_SQL = """SELECT DISTINCT table_name +FROM information_schema.columns +WHERE table_schema='public' +AND position('_' in table_name) <> 1 +AND position('pg_' in table_name) <> 1 +AND table_name IN (SELECT viewname FROM pg_views) +ORDER BY 1 +""" + + +VIEWS_INFO_SQL = """SELECT 'CREATE OR REPLACE VIEW ', viewname, ' AS ', definition FROM pg_views WHERE schemaname NOT IN ('pg_catalog', 'information_schema') +AND position('_' in viewname) <> 1 +AND position('pg_' in viewname) <> 1 ORDER BY viewname """ @@ -135,32 +237,76 @@ TRIGGERS_INFO_SQL = """SELECT event_object_table, trigger_name, action_orientation, action_timing, event_manipulation, action_statement FROM information_schema.triggers WHERE trigger_schema NOT IN ('pg_catalog', 'information_schema') +AND position('_' in event_object_table) <> 1 +AND position('pg_' in event_object_table) <> 1 ORDER BY 1, 2, 3, 4, 5""" +xxPROCEDURE_NAMES = """SELECT DISTINCT routine_name +FROM information_schema.routines +WHERE specific_schema NOT IN ('pg_catalog', 'information_schema') +AND position('_' in routine_name) <> 1 +AND position('pg_' in routine_name) <> 1 +AND position('dblink_' in routine_name) <> 1 +AND position('dex_' in routine_name) <> 1 +AND position('prsd_' in routine_name) <> 1 +AND position('rewrite' in routine_name) <> 1 +AND position('_tsquery' in routine_name) < 1 +AND routine_name NOT IN ('array_search', 'bytea_export', 'bytea_to_largeobject', +'headline', 'lexize', 'parse', 'rank', 'rank_cd', +'rewrite', 'rewrite_accum', 'rewrite_finish', +'set_curcfg', 'set_curdict', 'set_curprs', ) +ORDER BY 1""" + + +xxPROCEDURE_NAMES = """SELECT DISTINCT routine_name, external_language +FROM information_schema.routines +WHERE specific_schema NOT IN ('pg_catalog', 'information_schema') +AND position('_' in routine_name) <> 1 +AND position('pg_' in routine_name) <> 1 +AND position('ts_debug' in routine_name) <> 1 +AND external_language IN ('SQL', 'PLPGSQL', 'PLPYTHONU') +ORDER BY routine_name, external_language""" + + +PROCEDURE_NAMES = """SELECT proname, proname || '(' || pg_get_function_arguments(p.oid) || ')', lanname +FROM pg_catalog.pg_proc p +LEFT JOIN pg_catalog.pg_language l ON l.oid = prolang +JOIN pg_catalog.pg_namespace n + ON n.oid = p.pronamespace + WHERE n.nspname = 'public' +AND position('_' in p.proname) <> 1 +AND position('pg_' in p.proname) <> 1 +AND position('ts_debug' in p.proname) <> 1 +ORDER BY 1, 2, 3 +""" + _CONN = None TABLES = [] -def init_db_conn(connect_string, username, passwd): +def init_db_conn(connect_string, username, passwd, show_connection_info, show_version_info=True): """initializes db connections, can work with PyGres or psycopg2""" global _CONN try: dbinfo = connect_string - print(dbinfo) + if show_connection_info: + print(dbinfo) if USE_JYTHON: _CONN = zxJDBC.connect(connect_string, username, passwd, 'org.postgresql.Driver') elif '/' in connect_string: import odbc _CONN = odbc.odbc(connect_string) - print(_CONN) + if show_connection_info: + print(_CONN) elif connect_string.startswith('Driver='): import pyodbc # Driver={PostgreSQL};Server=IP address;Port=5432;Database=myDataBase;Uid=myUsername;Pwd=myPassword; # Driver={PostgreSQL};Server=isof-test64;Port=5435;Database=isof_stable;Uid=postgres;Pwd=postgres; _CONN = pyodbc.connect(connect_string) - print(_CONN) + if show_connection_info: + print(_CONN) else: # 'host:[port]:database:user:password' arr = connect_string.split(':') @@ -184,7 +330,8 @@ def init_db_conn(connect_string, username, passwd): else: sport = '' dsn = "host=%s %s dbname=%s user=%s password=%s" % (host, sport, dbname, user, passwd) - print(dsn) + if show_connection_info: + print(dsn) dbinfo = 'db: %s:%s' % (host, dbname) use_pgdb = 0 try: @@ -196,13 +343,18 @@ def init_db_conn(connect_string, username, passwd): except: raise exceptions.ImportError('No PostgreSQL library, install psycopg2 or PyGres!') if not _CONN: - print(dbinfo) + if show_connection_info: + print(dbinfo) if use_pgdb: _CONN = pgdb.connect(database=dbname, host=host, user=user, password=passwd) - print(_CONN) + if show_connection_info: + print(_CONN) else: _CONN = psycopg2.connect(dsn) - print(_CONN) + if show_connection_info: + print(_CONN) + if show_version_info: + add_ver_info(connect_string, username) except: ex = sys.exc_info() s = 'Exception: %s: %s\n%s' % (ex[0], ex[1], dbinfo) @@ -213,9 +365,18 @@ def init_db_conn(connect_string, username, passwd): def db_conn(): """access to global db connection""" + global _CONN return _CONN +def db_close(): + """closes global db connection""" + global _CONN + if _CONN: + _CONN.close() + _CONN = None + + def output_str(fout, line): """outputs line to fout trying various encodings in case of encoding errors""" if fout: @@ -228,11 +389,12 @@ def output_str(fout, line): ok = 0 for enc in DB_ENCODINGS: try: - line2 = line.decode(enc) - #fout.write(line2.encode(OUT_FILE_ENCODING)) - fout.write(line2) - ok = 1 - break + if not ok: + line2 = line.decode(enc) + #fout.write(line2.encode(OUT_FILE_ENCODING)) + fout.write(line2) + ok = 1 + break except (UnicodeDecodeError, UnicodeEncodeError): pass if not ok: @@ -246,7 +408,13 @@ def output_line(line, fout=None): """outputs line""" line = line.rstrip() output_str(fout, line) - output_str(sys.stdout, line) + output_str(OUTPUT_FILE, line) + + +def print_err(serr): + """println on stderr""" + sys.stderr.write('%s\n' % (serr)) + output_line('\nERROR! ERROR!\n%s\n' % (serr)) def select_qry(querystr): @@ -271,10 +439,19 @@ def fld2str(fld_v): return fld_v +def print_start_info(title): + output_line('\n\n') + output_line('--- %s (START) ---' % title) + + +def print_stop_info(title): + output_line('--- %s (END) ---' % title) + output_line('\n\n') + + def show_qry(title, querystr, fld_join='\t', row_separator=None): """prints rows from SELECT""" - output_line('\n\n') - output_line('--- %s ---' % title) + print_start_info(title) rs = select_qry(querystr) if rs: for row in rs: @@ -283,6 +460,7 @@ def show_qry(title, querystr, fld_join='\t', row_separator=None): output_line(row_separator) else: output_line(' -- NO DATA --') + print_stop_info(title) def show_qry_tables(title, querystr, fld_join='\t', row_separator=None): @@ -313,7 +491,33 @@ def show_qry_ex(querystr, table, fld_join='\t', row_separator=None): def init_session(): """place to change db session settings like locale""" - pass + global TABLES + TABLES = [] + + +def add_ver_info(connect_string, username): + """add version information""" + title = 'info' + print_start_info(title) + output_line('date: %s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) + output_line('connect string: %s' % (connect_string)) + output_line('user: %s' % (username)) + script_ver = __version__[5:-2] + output_line('created by: %s' % (script_ver)) + print_stop_info(title) + show_qry('DB version', DB_VERSION_SQL) + show_qry('DB name', 'SELECT current_catalog') + sel_info_option = '--ver-info-sql' + for s in sys.argv[1:]: + if s.startswith(sel_info_option): + sel = s[len(sel_info_option):].strip('=') + try: + show_qry(sel, sel) + except: + ex_info = traceback.format_exc() + serr = '\nSQL: %s\nException: %s\n' % (sel, ex_info) + print_err(serr) + break def show_tables(): @@ -328,17 +532,36 @@ def show_tables(): def show_primary_keys(): - show_qry('primary keys', PRIMARY_KEYS_INFO_SQL) + show_qry('primary keys idxs', PRIMARY_KEYS_INFO_SQL) + """ + title = 'primary keys' + print_start_info(title) + cur = db_conn().cursor() + cur.execute(PRIMARY_KEYS_INFO_SQL) + for row in cur.fetchall(): + pk_flds = [] + tblname = row[0] + if tblname in TABLES: + pk_idxs = '%s' % row[1] + for fld_idx in pk_idxs.split(): + sql = TABLE_COLUMN_NAME_BY_IDX % (tblname, fld_idx) + cur.execute(sql) + for row in cur.fetchall(): + pk_flds.append(row[0]) + output_line("%s\t(%s)" % (tblname, '-'.join(pk_flds))) + print_stop_info(title)""" def show_indexes(): - output_line('\n\n') - output_line('--- %s ---' % 'indexes') + title = 'indexes' + print_start_info(title) for tbl in TABLES: show_qry_ex(INDEXES_INFO_SQL, tbl) #show_qry('indexes columns', INDEXES_COLUMNS_INFO_SQL) - output_line('\n\n') - output_line('--- %s ---' % 'indexes columns') + print_stop_info(title) + + title = 'indexes columns' + print_start_info(title) cur = db_conn().cursor() for tbl in TABLES: #print @@ -353,6 +576,7 @@ def show_indexes(): cur.execute(sql) for row in cur.fetchall(): output_line("%s\t%s\t%s" % (tbl, idxname, row[0])) + print_stop_info(title) def show_foreign_keys(): @@ -364,9 +588,14 @@ def show_defaults(): def show_views(): + show_qry('view names', VIEW_NAMES_SQL) show_qry('views', VIEWS_INFO_SQL, '\n', '\n\n') +def show_types_stat(): + show_qry('column type count', SQL_TYPES) + + def get_arg_type(at): """returns type name from type id""" cur = db_conn().cursor() @@ -409,13 +638,12 @@ def get_subfields(subfield_fld): def show_procedures(): + show_qry('procedure names', PROCEDURE_NAMES) argtypes_dict = {} - output_line('\n\n --- procedures ---') + title = 'procedures' + print_start_info(title) cur = db_conn().cursor() - cur.execute("""SELECT DISTINCT routine_name - FROM information_schema.routines - WHERE specific_schema NOT IN ('pg_catalog', 'information_schema') - ORDER BY 1""") + cur.execute(PROCEDURE_NAMES) rows = cur.fetchall() for rt_row in rows: funname = rt_row[0] @@ -447,6 +675,7 @@ def show_procedures(): argtypes = [] argmodes = get_subfields(row[5]) argtypes_str = '%s' % row[1] + args = argtypes_str proc_body = '%s' % row[3] lang = row[6] argtypes_nrs = get_subfields(row[4]) @@ -456,17 +685,19 @@ def show_procedures(): argmodes = 'i' * len(argtypes_nrs) for at in argtypes_nrs: if at: - if not at in argtypes_dict.keys(): + if at not in argtypes_dict.keys(): argtypes_dict[at] = get_arg_type(at) ats = argtypes_dict[at] argtypes.append(ats) + if argtypes: + args = ', '.join(argtypes) argnames = get_subfields(row[2]) if argnames: args = ', '.join([join_arg(a, t, m) for (a, t, m) in zip(argnames, argtypes, argmodes)]) - output_line('\n\n -- >>> %s >>> --' % funname) - lines = proc_body.split('\n') - output_line('CREATE FUNCTION %s(%s) RETURNS %s\nAS $$' % (funname, args, ret_type)) + output_line('\n\n -- >>> %s(%s) [%s] >>> --' % (funname, args, lang)) + lines = proc_body.rstrip().split('\n') + output_line('CREATE OR REPLACE FUNCTION %s(%s) RETURNS %s\nAS $$' % (funname, args, ret_type)) was_line = 0 for line in lines: line = line.rstrip() @@ -475,8 +706,11 @@ def show_procedures(): output_line(line) output_line('$$') output_line(' LANGUAGE %s;' % (lang)) - output_line('\n\n -- <<< %s <<< --' % funname) + output_line(' -- <<< %s(%s) [%s] <<< --' % (funname, args, lang)) + output_line('') + output_line('') cur.close() + print_stop_info(title) def show_triggers(): @@ -484,43 +718,77 @@ def show_triggers(): def test(): - main('127.0.0.1:music:postgres:postgres') + info_filer = 'kifdvtp' + #main('127.0.0.1:music:postgres:postgres', None) + get_schema('test-baza.heuthesd:5494:pg_flugo:postgres:postgres', 'postgres', 'postgres', info_filer, True) -def main(connect_string): - username = None - passwd = None - if (len(sys.argv) > 2): - username = sys.argv[2] - if (len(sys.argv) > 3): - passwd = sys.argv[3] - if not init_db_conn(connect_string, username, passwd): +def get_schema(connect_string, username, passwd, info_filter, show_connection_info): + db_close() + t0 = time.time() + if not init_db_conn(connect_string, username, passwd, show_connection_info): print('Something is terribly wrong with db connection') else: init_session() show_tables() - if not TABLES_ONLY: - show_primary_keys() - show_indexes() - show_foreign_keys() - show_defaults() - show_views() - show_triggers() - show_procedures() - print('\n\n--- the end ---') - - -if '--version' in sys.argv: - print(__version__) -elif '--test' in sys.argv: - test() -elif '--help' in sys.argv: - print(USAGE) -elif __name__ == '__main__': - if '--tables_only' in sys.argv: - TABLES_ONLY = 1 - if len(sys.argv) < 2: - #print('arg len: %d' % (len(sys.argv))) + if not TABLES_ONLY and info_filter: + if 'k' in info_filter: + show_primary_keys() + if 'i' in info_filter: + show_indexes() + if 'f' in info_filter: + show_foreign_keys() + if 'd' in info_filter: + show_defaults() + if 'v' in info_filter: + show_views() + if 't' in info_filter: + show_triggers() + if 'p' in info_filter: + show_procedures() + if 'T' in info_filter: + show_types_stat() + t2 = time.time() + td = t2 - t0 + output_line('\n\nexecution time: %d min %d sec' % (divmod(td, 60))) + output_line('\n\n-- the end, filter [%s] --' % (info_filter)) + db_close() + + +def get_info_filter(): + result = '' + for s in sys.argv[1:]: + if s.startswith('--info-filter='): + _, result = s.split('=', 1) + if not result: + result = 'kifdvtpT' + return result + + +def main(): + if '--version' in sys.argv: + print(__version__) + return + if '--test' in sys.argv: + test() + return + if '--help' in sys.argv: print(USAGE) - else: - main(sys.argv[1]) + return + args = [x for x in sys.argv[1:] if not x.startswith('-')] + if not args: + print(USAGE) + return + connect_string = args[0] + username = None + passwd = None + if (len(args) > 1): + username = args[1] + if (len(args) > 2): + passwd = args[2] + info_filter = get_info_filter() + get_schema(connect_string, username, passwd, info_filter, False) + + +if __name__ == '__main__': + main() diff --git a/recipes/Python/576621_Dump_Informix_schema_to_text/recipe-576621.py b/recipes/Python/576621_Dump_Informix_schema_to_text/recipe-576621.py index 786574f7e..4226dde17 100644 --- a/recipes/Python/576621_Dump_Informix_schema_to_text/recipe-576621.py +++ b/recipes/Python/576621_Dump_Informix_schema_to_text/recipe-576621.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -__version__ = '$Id: schema_inf.py 1754 2014-02-14 08:57:52Z mn $' +__version__ = '$Id: schema_inf.py 3307 2017-11-29 06:18:57Z mn $' # export Informix schema to text using ODBC # usable to compare databases that should be the same @@ -22,11 +22,15 @@ # # Table names that start with '_' are ignored as they are temp tables # +# https://code.activestate.com/recipes/576621-dump-informix-schema-to-text/ +# # author: Michal Niklas USAGE = 'usage:\n\tschema_inf.py connect_string\n\t\tconnect string: odbc_database/user/password' +import re import sys +import time import traceback USE_JYTHON = 0 @@ -49,50 +53,76 @@ OUT_FILE_ENCODING = 'UTF8' +FILTER_TEMP = """tabtype='T' +AND systables.tabid >= 100 +AND tabname[1] <> '_' +""" + +if '--no-temp-tables' in sys.argv: + FILTER_TEMP += """AND tabname NOT LIKE '%tmp%' +AND tabname NOT LIKE '%temp%' +""" + +DB_VERSION_SQL = """SELECT FIRST 1 DBINFO('version','full') FROM systables""" TABLE_NAMES_SQL = """SELECT tabname FROM systables -WHERE tabtype='T' -AND tabid >= 100 -AND tabname[1] <> '_' -ORDER BY tabname""" +WHERE +%s +ORDER BY tabname""" % (FILTER_TEMP) + + +TABLE_COLUMNS_SQL = """SELECT tabname, colname +FROM syscolumns, systables +WHERE +%s +AND syscolumns.tabid = systables.tabid +ORDER BY tabname, colname +""" % (FILTER_TEMP) TABLE_INFO_SQL = """SELECT tabname, colname, colno, HEX(coltype), CASE MOD(coltype, 256) - WHEN 0 THEN 'char' - WHEN 1 THEN 'smallint' - WHEN 2 THEN 'integer' - WHEN 3 THEN 'float' - WHEN 4 THEN 'smallfloat' - WHEN 5 THEN 'decimal' - WHEN 6 THEN 'serial' - WHEN 7 THEN 'date' - WHEN 8 THEN 'money' - WHEN 9 THEN 'null' - WHEN 10 THEN 'datetime' - WHEN 11 THEN 'byte' - WHEN 12 THEN 'text' - WHEN 13 THEN 'varchar' - WHEN 14 THEN 'interval' - WHEN 15 THEN 'nchar' - WHEN 16 THEN 'nvarchar' - WHEN 17 THEN 'int8' - WHEN 18 THEN 'serial8' - WHEN 19 THEN 'set' - WHEN 20 THEN 'multiset' - WHEN 21 THEN 'list' - WHEN 22 THEN 'Unnamed ROW' - WHEN 40 THEN 'Variable-length' - WHEN 4118 THEN 'Named ROW' - ELSE '???' + WHEN 0 THEN 'char' + WHEN 1 THEN 'smallint' + WHEN 2 THEN 'integer' + WHEN 3 THEN 'float' + WHEN 4 THEN 'smallfloat' + WHEN 5 THEN 'decimal(' || (collength / 256)::int || ', ' || mod(collength, 256) || ')' + WHEN 6 THEN 'serial' + WHEN 7 THEN 'date' + WHEN 8 THEN 'money(' || (collength / 256)::int || ', ' || mod(collength, 256) || ')' + WHEN 9 THEN 'null' + WHEN 10 THEN 'datetime ' || substr(hex(collength), -2, 1) || ' to ' || substr(hex(collength), -1, 1) + WHEN 11 THEN 'byte' + WHEN 12 THEN 'text' + WHEN 13 THEN 'varchar' + WHEN 14 THEN 'interval' + WHEN 15 THEN 'nchar' + WHEN 16 THEN 'nvarchar' + WHEN 17 THEN 'int8' + WHEN 18 THEN 'serial8' + WHEN 19 THEN 'set' + WHEN 20 THEN 'multiset' + WHEN 21 THEN 'list' + WHEN 22 THEN 'row (unnamed)' + WHEN 23 THEN 'collection' + WHEN 40 THEN 'lvarchar fixed-length opaque types' + WHEN 41 THEN 'blob, boolean, clob variable-length opaque types' + WHEN 43 THEN 'lvarchar (client-side only)' + WHEN 45 THEN 'boolean' + WHEN 52 THEN 'bigint' + WHEN 53 THEN 'bigserial' + WHEN 2061 THEN 'idssecuritylabel' + WHEN 4118 THEN 'row (named)' + ELSE '???' END CASE, collength FROM syscolumns, systables -WHERE tabtype='T' -AND systables.tabid >= 100 -AND tabname[1] <> '_' +WHERE +%s AND syscolumns.tabid = systables.tabid ORDER BY tabname, colname -""" +""" % (FILTER_TEMP) + KEYS_INFO_SQL = """select tabname, a.colname column1, b.colname column2, c.colname column3, d.colname column4, e.colname column5, @@ -107,12 +137,11 @@ outer syscolumns i, outer syscolumns j, outer syscolumns k, outer syscolumns l, outer syscolumns m, outer syscolumns n, outer syscolumns o, outer syscolumns p -WHERE st.tabid >= 100 -AND tabtype='T' -AND tabname[1] <> '_' +WHERE +%s AND sc.tabid = st.tabid AND si.idxname = sc.idxname -AND sc.constrtype='%s' +AND sc.constrtype='%%s' and st.tabid = si.tabid and st.tabid = a.tabid and st.tabid = b.tabid @@ -147,7 +176,7 @@ and o.colno = part15 and p.colno = part16 ORDER BY tabname, column1, column2, column3, column4, column5 -""" +""" % (FILTER_TEMP.replace('systables.', 'st.')) INDEXES_INFO_SQL = """SELECT idxname, idxtype @@ -216,21 +245,18 @@ DEFAULTS_INFO_SQL = """SELECT tabname, colname, type, default FROM syscolumns, systables, sysdefaults -WHERE tabtype='T' -AND systables.tabid >= 100 -AND tabname[1] <> '_' +WHERE +%s AND syscolumns.tabid = systables.tabid AND sysdefaults.tabid = systables.tabid AND syscolumns.colno = sysdefaults.colno -ORDER BY tabname, colname""" +ORDER BY tabname, colname""" % (FILTER_TEMP) VIEWS_INFO_SQL = """SELECT tabname, tabid FROM systables -WHERE tabtype='V' -AND tabid >= 100 -AND tabname[1] <> '_' -""" +WHERE +%s""" % (FILTER_TEMP.replace("'T'", "'V'")) VIEWS_TEXT_SQL = """SELECT viewtext FROM sysviews @@ -241,11 +267,10 @@ TRIGGERS_INFO_SQL = """SELECT tabname, trigname, event FROM systables, systriggers -WHERE tabtype='T' -AND systables.tabid >= 100 -AND tabname[1] <> '_' +WHERE +%s AND systables.tabid = systriggers.tabid -ORDER BY tabname, trigname""" +ORDER BY tabname, trigname""" % (FILTER_TEMP) PROCEDURES_INFO_SQL = """SELECT procname, numargs, isproc, paramtypes::LVARCHAR, variant, handlesnulls, parallelizable @@ -254,6 +279,50 @@ ORDER BY procname, numargs, procid""" +SQL_TYPES = """SELECT COUNT(*)::int, +CASE MOD(coltype, 256) + WHEN 0 THEN 'char' + WHEN 1 THEN 'smallint' + WHEN 2 THEN 'integer' + WHEN 3 THEN 'float' + WHEN 4 THEN 'smallfloat' + WHEN 5 THEN 'decimal(' || (collength / 256)::int || ', ' || mod(collength, 256) || ')' + WHEN 6 THEN 'serial' + WHEN 7 THEN 'date' + WHEN 8 THEN 'money(' || (collength / 256)::int || ', ' || mod(collength, 256) || ')' + WHEN 9 THEN 'null' + WHEN 10 THEN 'datetime ' || substr(hex(collength), -2, 1) || ' to ' || substr(hex(collength), -1, 1) + WHEN 11 THEN 'byte' + WHEN 12 THEN 'text' + WHEN 13 THEN 'varchar' + WHEN 14 THEN 'interval' + WHEN 15 THEN 'nchar' + WHEN 16 THEN 'nvarchar' + WHEN 17 THEN 'int8' + WHEN 18 THEN 'serial8' + WHEN 19 THEN 'set' + WHEN 20 THEN 'multiset' + WHEN 21 THEN 'list' + WHEN 22 THEN 'row (unnamed)' + WHEN 23 THEN 'collection' + WHEN 40 THEN 'lvarchar fixed-length opaque types' + WHEN 41 THEN 'blob, boolean, clob variable-length opaque types' + WHEN 43 THEN 'lvarchar (client-side only)' + WHEN 45 THEN 'boolean' + WHEN 52 THEN 'bigint' + WHEN 53 THEN 'bigserial' + WHEN 2061 THEN 'idssecuritylabel' + WHEN 4118 THEN 'row (named)' + ELSE '???' +END CASE +FROM syscolumns, systables +WHERE +%s +AND syscolumns.tabid = systables.tabid +GROUP BY 2 +ORDER BY 1 desc, 2 +""" % (FILTER_TEMP) + _CONN = None _CONNECT_STRING = None @@ -276,20 +345,21 @@ def init_db_conn(connect_string, username, passwd): dbinfo = connect_string try: if USE_JYTHON: - print(dbinfo) + output_line(dbinfo) _CONN = zxJDBC.connect(connect_string, username, passwd, 'com.informix.jdbc.IfxDriver') else: (dbname, dbuser, _) = connect_string.split('/', 3) dbinfo = 'db: %s:%s' % (dbname, dbuser) try: _CONN = odbc.odbc(connect_string) - print(dbinfo) + output_line(dbinfo) except KeyboardInterrupt: raise + add_ver_info(connect_string, username) except: ex = sys.exc_info() s = 'Exception: %s: %s\n%s' % (ex[0], ex[1], dbinfo) - print(s) + output_line(s) return None return _CONN @@ -310,12 +380,42 @@ def show_db_error(querystr): """shows exception info""" ex = sys.exc_info() s = 'Exception: %s: %s\n\nSomething is terribly wrong with query:\n%s\n\n' % (ex[0], ex[1], querystr) - print('\n\n!!!\n\n%s\n\n!!!\n\n' % (s)) + output_line('\n\n!!!\n\n%s\n\n!!!\n\n' % (s)) sys.stderr.write(s) traceback.print_exc() reload_conn() +RE_DATETIME = re.compile(r'datetime (.) to (.)', re.IGNORECASE) + +IFX_DATETYPE = { + '0': "YEAR", + '2': "MONTH", + '4': "DAY", + '6': "HOUR", + '8': "MINUTE", + 'A': "SECOND", + 'B': "FRACTION(1)", + 'C': "FRACTION(2)", + 'D': "FRACTION(3)", + 'E': "FRACTION(4)", + 'F': "FRACTION(5)", + } + + +def linefilter_datetime(s): + """converts 'datetime 0 to 4' + to 'datetime YEAR to DAY' + """ + line = s + rx = RE_DATETIME.search(line) + if rx: + txt = line[rx.start(1):rx.end(2)] + txt2 = '%s to %s' % (IFX_DATETYPE[rx.group(1)], IFX_DATETYPE[rx.group(2)]) + line = line.replace(txt, txt2) + return line + + def output_str(fout, line): """outputs line to fout trying various encodings in case of encoding errors""" if fout: @@ -342,12 +442,20 @@ def output_str(fout, line): fout.flush() -def output_line(line): +def output_line(line, linefilter=None): """outputs line""" + if linefilter: + line = linefilter(line) line = line.rstrip() output_str(sys.stdout, line) +def print_err(serr): + """println on stderr""" + sys.stderr.write('%s\n' % (serr)) + output_line('\nERROR! ERROR!\n%s\n' % (serr)) + + def select_qry(querystr): """return rows from SELECT""" if querystr: @@ -375,28 +483,38 @@ def field_str(fld_value): return "???????" -def show_qry(title, querystr, fld_join='\t', row_separator=None): +def print_start_info(title): + output_line('\n\n') + output_line('--- %s (START) ---' % title) + + +def print_stop_info(title): + output_line('--- %s (END) ---' % title) + output_line('\n\n') + + +def show_qry(title, querystr, fld_join='\t', row_separator=None, linefilter=None): """prints rows from SELECT""" - print('\n\n') - print('--- %s ---' % title) + print_start_info(title) rs = select_qry(querystr) if rs: for row in rs: - output_line(fld_join.join([field_str(s) for s in row])) + output_line(fld_join.join([field_str(s) for s in row]), linefilter=linefilter) if row_separator: - print(row_separator) + output_line(row_separator) else: - print(' -- NO DATA --') + output_line(' -- NO DATA --') + print_stop_info(title) -def show_qry_ex(querystr, table, fld_join='\t', row_separator=None): +def show_qry_ex(querystr, table, fld_join='\t', row_separator=None, linefilter=None): """like show_qry() but with table name as first column""" rs = select_qry(querystr % table) if rs: for row in rs: - output_line("%s%s%s" % (table, fld_join, fld_join.join([field_str(s) for s in row]))) + output_line("%s%s%s" % (table, fld_join, fld_join.join([field_str(s) for s in row])), linefilter=linefilter) if row_separator: - print(row_separator) + output_line(row_separator) def init_session(): @@ -404,6 +522,31 @@ def init_session(): pass +def add_ver_info(connect_string, username): + """add version information""" + title = 'info' + print_start_info(title) + output_line('date: %s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) + output_line('connect string: %s' % (connect_string)) + output_line('user: %s' % (username)) + script_ver = __version__[5:-2] + output_line('created by: %s' % (script_ver)) + print_stop_info(title) + show_qry('DB version', DB_VERSION_SQL) + show_qry('DB name', "SELECT DBINFO('dbname') FROM systables WHERE tabid = 1") + sel_info_option = '--ver-info-sql' + for s in sys.argv[1:]: + if s.startswith(sel_info_option): + sel = s[len(sel_info_option):].strip('=') + try: + show_qry(sel, sel) + except: + ex_info = traceback.format_exc() + serr = '\nSQL: %s\nException: %s\n' % (sel, ex_info) + print_err(serr) + break + + def show_tables(): """prints table names""" cur = db_conn().cursor() @@ -413,7 +556,8 @@ def show_tables(): TABLES.append(row[0]) cur.close() show_qry('tables', TABLE_NAMES_SQL) - show_qry('columns', TABLE_INFO_SQL) + show_qry('table columns', TABLE_COLUMNS_SQL) + show_qry('columns', TABLE_INFO_SQL, linefilter=linefilter_datetime) def show_primary_keys(): @@ -423,15 +567,18 @@ def show_primary_keys(): def show_indexes(): """print indexes""" - print('\n\n') - print('--- %s ---' % 'indexes') + title = 'indexes' + print_start_info(title) for tbl in TABLES: show_qry_ex(INDEXES_INFO_SQL, tbl) #show_qry('indexes columns', INDEXES_COLUMNS_INFO_SQL) - print('\n\n') - print('--- %s ---' % 'indexes columns') + print_stop_info(title) + + title = 'indexes columns' + print_start_info(title) for tbl in TABLES: show_qry_ex(INDEXES_COLUMNS_INFO_SQL, tbl) + print_stop_info(title) def show_foreign_keys(): @@ -446,8 +593,8 @@ def show_defaults(): def show_views(): """print views""" - print('\n\n') - print('--- %s ---' % 'views') + title = 'views' + print_start_info(title) cur = db_conn().cursor() try: cur.execute(VIEWS_INFO_SQL) @@ -458,17 +605,22 @@ def show_views(): try: cur2 = db_conn().cursor() cur2.execute(querystr) - print(tabname) + output_line(tabname) vt = [] for row2 in cur2.fetchall(): vt.append(row2[0]) vtt = ''.join(vt) output_line(vtt.rstrip()) - print('') + output_line('') except: show_db_error(querystr) except: show_db_error(VIEWS_INFO_SQL) + print_stop_info(title) + + +def show_types_stat(): + show_qry('column type count', SQL_TYPES, linefilter=linefilter_datetime) def get_body(qry): @@ -485,8 +637,8 @@ def get_body(qry): def show_procedures(): """show procedures and functions""" show_qry('procedures/functions', PROCEDURES_INFO_SQL) - print('\n\n') - print('--- %s ---' % 'procedures/functions bodies') + title = 'procedures/functions bodies' + print_start_info(title) querystr1 = """SELECT procid, procname FROM sysprocedures WHERE internal='f' AND mode IN ('D', 'd', 'O', 'o') @@ -502,17 +654,17 @@ def show_procedures(): for row in rs: funname = row[1] body = get_body(querystr2 % row[0]) - print('\n\n -- >>> %s >>> --' % funname) + output_line('\n\n -- >>> %s >>> --' % funname) output_line(body) - print('\n\n -- <<< %s <<< --' % funname) - print('--- ---') + output_line('\n\n -- <<< %s <<< --' % funname) + print_stop_info(title) def show_triggers(): """show triggers""" show_qry('triggers', TRIGGERS_INFO_SQL) - print('\n\n') - print('--- %s ---' % 'triggers bodies') + title = 'triggers bodies' + print_start_info(title) querystr1 = """SELECT trigid, tabname, trigname FROM systables, systriggers WHERE tabtype='T' @@ -537,41 +689,70 @@ def show_triggers(): for row in rs: body_def = get_body(querystr2 % trigid) body_txt = get_body(querystr3 % trigid) - print('\n\n -- >>> %s >>> --' % trigname) + output_line('\n\n -- >>> %s >>> --' % trigname) output_line(body_def) output_line(body_txt) - print('\n\n -- <<< %s <<< --' % trigname) - print('--- ---') + output_line('\n\n -- <<< %s <<< --' % trigname) + print_stop_info(title) + + +def get_info_filter(): + result = '' + for s in sys.argv[1:]: + if s.startswith('--info-filter='): + _, result = s.split('=', 1) + if not result: + result = 'kifdvtpT' + return result def main(): """main""" - connect_string = sys.argv[1] + if '--version' in sys.argv: + print(__version__) + return + args = [x for x in sys.argv[1:] if not x.startswith('-')] + if not args: + print(USAGE) + return + connect_string = args[0] username = None passwd = None - if (len(sys.argv) > 2): - username = sys.argv[2] - if (len(sys.argv) > 3): - passwd = sys.argv[3] + if (len(args) > 1): + username = args[1] + if (len(args) > 2): + passwd = args[2] if not init_db_conn(connect_string, username, passwd): - print('Something is terribly wrong with db connection') + output_line('Something is terribly wrong with db connection') else: + tables_only = '--tables-only' in sys.argv + info_filter = get_info_filter() + t0 = time.time() init_session() show_tables() - show_primary_keys() - show_indexes() - show_foreign_keys() - show_defaults() - show_views() - show_triggers() - show_procedures() - print('--- the end ---') - - -if '--version' in sys.argv: - print(__version__) -elif __name__ == '__main__': - if len(sys.argv) < 2: - print(USAGE) - else: - main() + if not tables_only and info_filter: + if 'k' in info_filter: + show_primary_keys() + if 'i' in info_filter: + show_indexes() + if 'f' in info_filter: + show_foreign_keys() + if 'd' in info_filter: + show_defaults() + if 'v' in info_filter: + show_views() + if 't' in info_filter: + show_triggers() + if 'p' in info_filter: + show_procedures() + if 'T' in info_filter: + show_types_stat() + + t2 = time.time() + td = t2 - t0 + output_line('\n\nexecution time: %d min %d sec' % (divmod(td, 60))) + output_line('\n\n-- the end, filter [%s] --' % (info_filter)) + + +if __name__ == '__main__': + main() From 744c1a3f65c2043b7d5302de677cc5b87933777b Mon Sep 17 00:00:00 2001 From: Alan Young Date: Sat, 23 Dec 2017 11:48:06 -0700 Subject: [PATCH 05/11] mktemp is a better way to manage tmp files --- recipes/Bash/578661_find__grep/recipe-578661.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/Bash/578661_find__grep/recipe-578661.sh b/recipes/Bash/578661_find__grep/recipe-578661.sh index 3846e2bee..62773a86f 100644 --- a/recipes/Bash/578661_find__grep/recipe-578661.sh +++ b/recipes/Bash/578661_find__grep/recipe-578661.sh @@ -1,6 +1,6 @@ #!/usr/bin/env zsh -tmp_file="/tmp/_unsorted_find_n_grep" +tmp_file="$(mktemp)" echo "--- find <$2> in <$1>" > $tmp_file find . -follow -iname "$1" | while read file From 610d487c85cbfc3d68c1027df8c1ca04d8a6f541 Mon Sep 17 00:00:00 2001 From: Alan Young Date: Sat, 23 Dec 2017 11:50:41 -0700 Subject: [PATCH 06/11] added an alias for tree command if it doesnt exist on the system --- recipes/Bash/tree_alias/LICENSE.md | 1 + recipes/Bash/tree_alias/README.md | 8 ++++++++ recipes/Bash/tree_alias/tree | 10 ++++++++++ 3 files changed, 19 insertions(+) create mode 100644 recipes/Bash/tree_alias/LICENSE.md create mode 100644 recipes/Bash/tree_alias/README.md create mode 100644 recipes/Bash/tree_alias/tree diff --git a/recipes/Bash/tree_alias/LICENSE.md b/recipes/Bash/tree_alias/LICENSE.md new file mode 100644 index 000000000..09dcacce0 --- /dev/null +++ b/recipes/Bash/tree_alias/LICENSE.md @@ -0,0 +1 @@ +This work is released to the public domain under the CC0 (Creative Commons) license terms: https://creativecommons.org/publicdomain/zero/1.0/legalcode \ No newline at end of file diff --git a/recipes/Bash/tree_alias/README.md b/recipes/Bash/tree_alias/README.md new file mode 100644 index 000000000..5d594ab7e --- /dev/null +++ b/recipes/Bash/tree_alias/README.md @@ -0,0 +1,8 @@ +### An alias for the tree command + +Author: Unknown +Submitter: Alan Young (harleypig) + +If the tree command does not exist on your system, this alias will fake it. + +I got this from a co-worker a long time ago. I don't remember when, or who it was. If it was you, let me know and I'll update this. diff --git a/recipes/Bash/tree_alias/tree b/recipes/Bash/tree_alias/tree new file mode 100644 index 000000000..d9282f262 --- /dev/null +++ b/recipes/Bash/tree_alias/tree @@ -0,0 +1,10 @@ +#!/bin/bash + +# I got this from a co-worker many moons ago. Unfortunately, I don't remember +# who. Basically, if a tree program is not installed, fake it with this. + +if ! command -v tree > /dev/null 2>&1; then + + alias tree="find . -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'" + +fi From acde9bbeb7337c821934d8649ca203c9e3192303 Mon Sep 17 00:00:00 2001 From: Geoff Crompton Date: Thu, 15 Mar 2018 11:42:03 +1100 Subject: [PATCH 07/11] Removing 278260, it's a duplicate of 278259 --- .../Python/278260_Dictionary_Tools/LICENSE.md | 3 - .../Python/278260_Dictionary_Tools/README.md | 6 - .../278260_Dictionary_Tools/recipe-278260.py | 291 ------------------ 3 files changed, 300 deletions(-) delete mode 100644 recipes/Python/278260_Dictionary_Tools/LICENSE.md delete mode 100644 recipes/Python/278260_Dictionary_Tools/README.md delete mode 100644 recipes/Python/278260_Dictionary_Tools/recipe-278260.py diff --git a/recipes/Python/278260_Dictionary_Tools/LICENSE.md b/recipes/Python/278260_Dictionary_Tools/LICENSE.md deleted file mode 100644 index 86fe1ce11..000000000 --- a/recipes/Python/278260_Dictionary_Tools/LICENSE.md +++ /dev/null @@ -1,3 +0,0 @@ -Copyright 2004 Runsun Pan - -The software is licensed according to the terms of the PSF (Python Software Foundation) license found here: http://www.python.org/psf/license/ \ No newline at end of file diff --git a/recipes/Python/278260_Dictionary_Tools/README.md b/recipes/Python/278260_Dictionary_Tools/README.md deleted file mode 100644 index 61a762b68..000000000 --- a/recipes/Python/278260_Dictionary_Tools/README.md +++ /dev/null @@ -1,6 +0,0 @@ -## Dictionary Tools -Originally published: 2004-04-14 14:06:08 -Last updated: 2004-04-14 14:06:08 -Author: Runsun Pan - -A collection of some dictionary tools \ No newline at end of file diff --git a/recipes/Python/278260_Dictionary_Tools/recipe-278260.py b/recipes/Python/278260_Dictionary_Tools/recipe-278260.py deleted file mode 100644 index 5a48beb42..000000000 --- a/recipes/Python/278260_Dictionary_Tools/recipe-278260.py +++ /dev/null @@ -1,291 +0,0 @@ -#======================================================================== - -def sumDict(d): - return reduce(lambda x,y:x+y, d.values()) - -#======================================================================== - -def avgDict(d): - return reduce(lambda x,y:x+y, d.values()) / (len(d)*1.0) - -#======================================================================== - -def mkDict(keyList, valueList, cond=None): - ''' - Make a new dict out of two lists. The first list provides keys, - the 2nd provides values. The cond is a function taking 2 arguments: - key and value. Example: lambda k,v:v%2==0 - A valid item means its cond(key, value) is true. Only valid items - are included in the returned dict. - - >>> a - [0, 1, 2, 3, 4] - - >>> b= [chr(x) for x in range(65,70)] - >>> b - ['A', 'B', 'C', 'D', 'E'] - - >>> mkDict(b,a) - {'A': 0, 'C': 2, 'B': 1, 'E': 4, 'D': 3} - - >>> mkDict(b,a,lambda x,y:y%2==0) - {'A': 0, 'C': 2, 'E': 4} - - Note: - mkDict(a,b) (without the conditional check) is equavilant to - dict(zip(a,b)), which is much faster. If you know there's no - conditional check, better use dict(zip(a,b)) instead. - ''' - - if cond==None: - return dict(zip(keyList, valueList)) - else: - return dict( [(k,v) for k,v in zip(keyList, valueList) if cond(k,v)] ) - -#======================================================================== - -def trimDict(aDict, cond=(lambda k,v:1)): - ''' Return a new dict in which its items whose cond(k,v) == true - are removed (discarded) from aDict. - The cond is a function taking 2 arguments: key and value - - >>> g - {'A': 0, 'C': 2, 'B': 1, 'E': 4, 'D': 3} - >>> trimDict(g, lambda x,y:y%2!=0) - {'A': 0, 'C': 2, 'E': 4} - - ''' - tmp={} - [tmp.setdefault(k,v) for k,v in aDict.items() if not cond(k,v)] - return tmp.copy() - -#======================================================================== - -def reDict(aDict, func, cond=(lambda k,v:1), delUnchanged=0): - ''' - Given aDict, return a new dict with valid items (= items whose - cond(key, value) is true) been modified by the function func. - if delUnchanged, then those invalid items are excluded from - the returned dict. - - >>> d - {'B': 1, 'D': 3} - >>> reDict(d, lambda x:x**2) - {'B': 1, 'D': 9} - >>> g - {'A': 0, 'C': 2, 'B': 1, 'E': 4, 'D': 3} - >>> reDict(g, lambda x:-x, lambda k,v: v%2==0) - {'A': 0, 'C': -2, 'B': 1, 'E': -4, 'D': 3} - >>> reDict(g, lambda x:-x, lambda k,v: v%2==0, delUnchanged=1) - {'A': 0, 'C': -2, 'E': -4} - - ''' - tmp={} - if delUnchanged: - [tmp.setdefault(k,func(v)) for k,v in aDict.items() if cond(k,v)] - else: - [tmp.setdefault(k,(cond(k,v) and func(v) or v)) for k,v in aDict.items()] - return tmp.copy() - -#======================================================================== - -def normDict(d, normalizeTo=1): - ''' Normalize the values of d by setting the largest value in d.values - to normalizeTo. The return dict will have it's values spreading - between 0~normalizeTo''' - ks = d.keys() - vs = d.values() - vMax = max(vs) - vs= [ x/(vMax*1.0)*normalizeTo for x in vs] - return dict(zip(ks, vs)) - -#======================================================================== - -def normDictSumTo(d, sumTo=1): - ''' Normalize the values of d to sumTo. The returned dict will - have value sum = sumTo - ''' - ks = d.keys() - vs = d.values() - sum = reduce(lambda x,y:x+y, vs) - vs= [ x/(sum*1.0)*sumTo for x in vs] - return dict(zip(ks, vs)) - -#======================================================================== - -def accumDict(d, normalizeTo=None): - ''' if d.values=[1,2,3,4,5]: - accumDict(d).values ==> [1, 3, 6, 10, 15] - if d.values=[0.25, 0.25, 0.25, 0.25] - accumDict(d).values ==> [0.25, 0.50, 0.75, 1.00] - - Required: odict() - Ordered Dictionary Class - Dave Benjamin - http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/161403 - ''' - ks = d.keys() - vs = d.values() - if normalizeTo: vs = normListSumTo(vs, sumTo=normalizeTo) - - r = range(1, len(vs)) - newList=[vs[0]] - for i in r: - newList.append( newList[-1]+ vs[i] ) - - odic = odict() - for i in range(len(newList)): - odic[ks[i]] =newList[i] - return odic - -#======================================================================== - -def fallInDict(val, aDict, baseValue=0): - ''' - oo: - { - 'a':3, - 'b':5, - 'c':2 - } - accumDict(oo): - { - 'a':3, - 'b':8, - 'c':10 - } - fallInDict(0, oo)=0 - fallInDict(1, oo)=0 - fallInDict(2, oo)=0 - fallInDict(3, oo)=0 - fallInDict(4, oo)=1 - fallInDict(5, oo)=1 - fallInDict(6, oo)=1 - fallInDict(7, oo)=1 - fallInDict(8, oo)=1 - fallInDict(9, oo)=2 - fallInDict(10, oo)=2 - - When val out of range: - - fallInDict(-1, oo)= -1 - fallInDict(11, oo)= -2 - fallInDict(12, oo)= -2 - ''' - - if valacd.values()[-1]: return -2 - - ra = range(len(acd)) - - for i in ra: - vInDict = acd.values()[i] - if val <= vInDict: break - - return aDict.keys()[i] - -#======================================================================== - -def randomPickDict(d): - ''' given a diictionary d, with all values are numbers, - randomly pick an item and return it's key according - to the percentage of its values''' - ks = d.keys() - vs = accumList(d.values(),1) - return ks[ findIndex( vs, random.random()) ] - -#======================================================================== - -def sumInDict(L): - ''' Given a list L: - [ {'S': [s11,s12,s13], 'V':[v11,v12,v13], ...}, - {'S': [s21,s22,s23], 'V':[v21,v22,v23], ...}, - {'S': [s31,s32,s33], 'V':[v31,v32,v33], ...}, - ... ] - - Return a dict: - {'S': [s1,s2,s3], 'V':[v1,v2,v3], ...} - - where s1 = s11 + s21 + s31 + ... - s2 = s12 + s22 + s32 + ... - ''' - keys = L[0].keys() - new={} - for k in keys: - new[k] = [] # Init 'S':[] - for x in L: # x = {'S': [s11,s12,s13], 'V':[v11,v12,v13], ...} - y = x[k] # y = [s11,s12,s13] - new[k].append(y) - - ''' new[k] = [ [s11,s12,s13],[s21,s22,s23],[s31,s32,s33], ...] - Then use sumInList=> {'S': [s1,s2,s3],'V': [v1,v2,v3],... }''' - new[k] = sumInList(new[k]) - return new - -#======================================================================== - -def avgInDict(L): - ''' Given a list L: - [ {'S': [s11,s12,s13], 'V':[v11,v12,v13], ...}, - {'S': [s21,s22,s23], 'V':[v21,v22,v23], ...}, - {'S': [s31,s32,s33], 'V':[v31,v32,v33], ...}, - ... ] - - Return a dict: - {'S': [s1,s2,s3], 'V':[v1,v2,v3], ...} - - where s1 = avg of (s11 + s21 + s31 + ...) - s2 = avg of (s12 + s22 + s32 + ...) - ''' - ''' First turn L into: - {'S': [ [s11,s12,s13],[s21,s22,s23],[s31,s32,s33], ...], - 'V': [ [v11,v12,v13],[v21,v22,v23],[v31,v32,v33], ...], - ... } ''' - - keys = L[0].keys() - new={} - for k in keys: - new[k] = [] # Init 'S':[] - for x in L: # x = {'S': [s11,s12,s13], 'V':[v11,v12,v13], ...} - y = x[k] # y = [s11,s12,s13] - new[k].append(y) - - ''' new[k] = [ [s11,s12,s13],[s21,s22,s23],[s31,s32,s33], ...] - Then use avgInList=> {'S': [s1,s2,s3],'V': [v1,v2,v3],... }''' - new[k] = avgInList(new[k]) - return new - -#======================================================================== - -def sumInDict2(D): - ''' Given a dict: - {'A':{'S': [s11,s12,s13], 'V':[v11,v12,v13], ...}, - 'B':{'S': [s21,s22,s23], 'V':[v21,v22,v23], ...}, - 'C':{'S': [s31,s32,s33], 'V':[v31,v32,v33], ...}, ... - } - Return a dict: - {'S': [s1,s2,s3], 'V':[v1,v2,v3], ...} - - where s1 = s11 + s21 + s31 + ... - s2 = s12 + s22 + s32 + ... - ''' - return sumInDict(D.values()) - -#======================================================================== - -def avgInDict2(D): - ''' Given a dict: - {'A':{'S': [s11,s12,s13], 'V':[v11,v12,v13], ...}, - 'B':{'S': [s21,s22,s23], 'V':[v21,v22,v23], ...}, - 'C':{'S': [s31,s32,s33], 'V':[v31,v32,v33], ...}, ... - } - Return a dict: - {'S': [s1,s2,s3], 'V':[v1,v2,v3], ...} - - where s1 = avg of (s11 + s21 + s31 + ...) - s2 = avg of (s12 + s22 + s32 + ...) - ''' - return avgInDict(D.values()) - -#======================================================================== From 61a74f5f93da087d27c70b2efe779ac6bd2a3b4f Mon Sep 17 00:00:00 2001 From: Jeff Rouse Date: Tue, 2 Jul 2019 23:44:37 -0700 Subject: [PATCH 08/11] Changed extension to denote a java file. --- .../{recipe-576955.js => recipe-576955.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename recipes/JavaScript/576955_Incrementing_counter/{recipe-576955.js => recipe-576955.java} (100%) diff --git a/recipes/JavaScript/576955_Incrementing_counter/recipe-576955.js b/recipes/JavaScript/576955_Incrementing_counter/recipe-576955.java similarity index 100% rename from recipes/JavaScript/576955_Incrementing_counter/recipe-576955.js rename to recipes/JavaScript/576955_Incrementing_counter/recipe-576955.java From 5b83ef48a845deffe0f8596724678e73390a3885 Mon Sep 17 00:00:00 2001 From: Tammy DiPrima Date: Mon, 11 May 2020 12:08:15 -0400 Subject: [PATCH 09/11] fixed missing import statement 'import re' --- .../577468_Minimalistic_PostgreSQL_console/recipe-577468.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/Python/577468_Minimalistic_PostgreSQL_console/recipe-577468.py b/recipes/Python/577468_Minimalistic_PostgreSQL_console/recipe-577468.py index 53eb233f4..18c4e9769 100644 --- a/recipes/Python/577468_Minimalistic_PostgreSQL_console/recipe-577468.py +++ b/recipes/Python/577468_Minimalistic_PostgreSQL_console/recipe-577468.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import re import sys import psycopg2 import psycopg2.extras From db8c96c09c06d52b1ad4b0a6ff7f001f3d2241d6 Mon Sep 17 00:00:00 2001 From: swaroop998 <72223625+swaroop998@users.noreply.github.com> Date: Fri, 2 Oct 2020 00:08:28 +0530 Subject: [PATCH 10/11] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c7f9c4eb7..28c4eaf7a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ActiveState Code Recipes -Welcome to the ActiveState code recipes repo! We have migrated all of the great content from code.activestate.com to its new +Welcome to the ActiveState Code recipes repo! We have migrated all of the great content from code.activestate.com to its new forever-home here at GitHub. This makes it easier for everyone to submit new recipes, contribute code and integrate all the great information into their own projects. @@ -18,7 +18,7 @@ You can find the master index of all recipes over at the [wiki](https://github.c *Coming Soon: Search Functionality!* -## How do I submit a new recipe? +## How could I submit a new recipe? 1. Fork this repo. 2. Create your new recipe in the correct language subfolder (create a new folder if it doesn't already exist). From e6378c55515c45eb8eab07ca64c59ddefba9bf94 Mon Sep 17 00:00:00 2001 From: Jeff Rouse Date: Mon, 12 Oct 2020 16:31:24 -0700 Subject: [PATCH 11/11] Removed short form for between for inline docs. --- .../Python/578112_Bresenhams_line_algorithm/recipe-578112.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/Python/578112_Bresenhams_line_algorithm/recipe-578112.py b/recipes/Python/578112_Bresenhams_line_algorithm/recipe-578112.py index 8838ad588..dffe22406 100644 --- a/recipes/Python/578112_Bresenhams_line_algorithm/recipe-578112.py +++ b/recipes/Python/578112_Bresenhams_line_algorithm/recipe-578112.py @@ -68,8 +68,8 @@ def _bresenhamlines(start, end, max_iter): def bresenhamline(start, end, max_iter=5): """ - Returns a list of points from (start, end] by ray tracing a line b/w the - points. + Returns a list of points from (start, end] by ray tracing a line between + the points. Parameters: start: An array of start points (number of points x dimension) end: An end points (1 x dimension)