import os
import platform
import sys
import time
from datetime import datetime
import re
import glob
import builtins
import traceback
import getpass
import openpyxl
from importlib import import_module

try:
    import traceback
    from pygments import highlight
    from pygments.lexers import get_lexer_by_name
    from pygments.formatters import TerminalFormatter

    use_colored_traceback = True
except:
    use_colored_traceback = False

from tklib.tkobject import tkObject
from tklib.tkparams import tkParams
from tklib.tkvariousdata import tkVariousData
from tklib.tkfile import open_chardet
from tklib.tkplugin import tkModule
from tklib.tkutils import set_exception, terminate, getarg, getintarg, getfloatarg, pint, pfloat, pconv, pconv_by_type, replace_path
from tklib.tkutils import convert_by_vars, conv_float, save_csv, del_quote, split_file_path, val2str

#from tklib.tksci.tkFit_object import save_fit_config


#=========================
# Application base class
#=========================


def save_fit_config(vars, outfile, 
            labels = ["varname", "unit", "pk_scale", "optid", "x0", "dx", "kmin", "kmax", "kpenalty"],
            section = "Parameters"):
    if vars.hasattr('print_variables'): vars.print_variables()

    ext = get_ext(outfile).lower()
    if ext in ['.csv', '.xlsx']:
        data_list = [getattr(vars, l) for l in labels]
        return tkVariousData().save(outfile, data_list = data_list, labels = labels)

    return None


class tkApplication(tkObject):
    def __init__(self, usage_str = "", suppress_usage = '', _globals = None, locals = None, 
                 use_user_inifile = False, inifile_dirs = ['ini'], create_inidir = False,
                 save_time = True,
                 **args):
        super().__init__()
        
        self.os_name          = platform.system()
        self.args_opt_inf     = {}
        self.args_idx_inf     = []
        self.args_vars        = None

        self.suppress_usage = suppress_usage
        self.set_usage(usage_str, suppress_usage = suppress_usage)

        self.phrases     = {}
        self.save_time   = save_time
        self.time_stamps = {}
        if self.save_time: self.print_time(label = "startup")

        self.user            = getpass.getuser()
        self.user_nospace    = self.user.replace(' ', '')

        self.argv            = sys.argv
        self.script_path     = sys.argv[0]
        self.script_fullpath = os.path.abspath(self.script_path)
        dirname, basename, filebody, ext = split_file_path(self.script_fullpath)
        self.script_dir      = dirname
        self.script_filebody = filebody
#        self.script_dir      = os.path.splitext(self.script_fullpath)[0]

        self.ini_dir = self.get_inifile_dir(use_user_inifile = use_user_inifile,
                             inifile_dirs = inifile_dirs, create_inidir = create_inidir,
                              defval = self.script_dir)
        self.inifile = self.get_inifile_path(self.ini_dir, use_user_inifile = use_user_inifile)

        self.redirects      = []
        self.print_original = print
        
        self.globals     = globals
        self.locals      = locals
        self.modules     = []

# Configuration parameters
        self.configparams   = tkParams(app = self)
        self.config         = self.configparams
        self.config.lang    = 'en'
        self.config.inifile = self.inifile
        self.config.ini_dir = os.path.dirname(os.path.abspath(self.config.inifile))

# Application global variables
        self.params = tkParams(app = self)

# Script variables
        self.svars  = tkParams(app = self)

        self.update(**args)

    def __del__(self):
        if self.get('redirects', None):
            for target in self.redirects:
                if type(target) is str:
                    pass
                else:
                    try:
                        target.close()
                    except:
                        pass

#        super(tkObject, self).__del__()

    def __str__(self):
        return self.ClassPath()

    def nohup(self, flag):
        if flag: signal.signal(signal.SIGHUP, signal.SIG_IGN)

    def daemonize(self, stdin_path = '/dev/null', stdout_path = '/dev/null', stderr_path = '/dev/null', print_level = 0):
        if os.name != 'posix':
            self.terminate(f"Error in daemonize(): running on [{os.name}] but daemonize() is valid for linux-based OS.\n", pause = True)

        if print_level:
            print("daemonize the process:")
            print(f"  stdin is redirected to [{stdin_path}]")
            print(f"  stdout is redirected to [{stdout_path}]")
            print(f"  stderr is redirected to [{stderr_path}]")

        if os.fork() > 0: sys.exit()
        os.setsid()
        if os.fork() > 0: sys.exit()
        
        sys.stdout.flush()
        sys.stderr.flush()
        with open(stdin_path, 'r') as f:
            os.dup2(f.fileno(), sys.stdin.fileno())
        with open(stdout_path, 'a+') as f:
            os.dup2(f.fileno(), sys.stdout.fileno())
        with open(stderr_path, 'a+') as f:
            os.dup2(f.fileno(), sys.stderr.fileno())

    def raise_error(self, desc = "custom error"):
        class CustomError(Exception):
            pass

        raise CustomError(desc)

    def set_exception(self, func = None, print_level = 1):
        set_exception(func, print_level = print_level)

    def exception_hook(self, type, value, tb, output = 'stderr', display_type = 'colored', pause_at_terminate = True):
        if not use_colored_traceback:
            return

        tbtext    = ''.join(traceback.format_exception(type, value, tb))
        lexer     = get_lexer_by_name("pytb", stripall=True)
        formatter = TerminalFormatter()
        if display_type == 'colored':
            if output == 'stderr':
                sys.stderr.write("\n" + highlight(tbtext, lexer, formatter))
            else:
                print("\n" + highlight(tbtext, lexer, formatter))
        elif display_type == 'plain':
            if output == 'stderr':
                sys.stderr.write("\n" + tbtext)
            else:
                print("\n" + tbtext)
        else:
            print("")
            print(f"Error in exception_hook(): Invalid display_type [{display_type}]")

        if pause_at_terminate:
            input("Press ENTER to terminate>> ")
#            try:
#                input("Press ENTER to terminate>> ")
#            except:
#                pass

    def redirect_exception(self, hook_func = None, output = 'stdout', display_type = 'colored', pause_at_terminate = True):
        if not use_colored_traceback:
            return

        if hook_func is None:
            hook_func = lambda type, value, tb: \
                        self.exception_hook(type, value, tb, output = output, display_type = display_type, pause_at_terminate = pause_at_terminate)

        sys.excepthook = hook_func

    def print_redict(self, *args, **kwargs):
        return self.redirect(*args, **kwargs)

    def print_warning(self, *args, **kwargs):
        return self.print_redirect(*args, console_out = True, **kwargs)

    def print_redirect(self, *args, **kwargs):
        if kwargs.get("console_out", None) and 'stdout' not in self.redirects:
            del kwargs['console_out']
            self.print_original(*args, **kwargs)
        if kwargs.get("console_out", None) is not None:
            del kwargs['console_out']
        
        for target in self.redirects:
            if type(target) is str:
                if target == 'stdout':
                    if 'file' in self.display_type_traceback:
                        for arg in args:
                            args = re.sub(r'\x1b\[[\d;]+m', '', args, flags = re.MULTILINE)

                    try:
                        self.print_original(*args, **kwargs)
                    except UnicodeEncodeError as e:
                        print(f"Error in print_original(): UnicodeEncodeError: {e}. Continue")
                    except Exceptions as e:
                        print(f"Error in print_original(): {e}. Continue")

                continue

            fp = target
            if fp is None:
                return

            if kwargs.get('end', None) is None:
                newline = True
            else:
                newline = False
                del kwargs['end']

            for a in args:
                if self.display_type_traceback == 'file':
                    a = re.sub(r'\x1b\[[\d;]+m', '', a, flags = re.MULTILINE)

                try:
                    fp.write("{}".format(a), **kwargs)
                except:
                    pass

            if newline:
                fp.write("\n")
#                fp.write("\n", **kwargs)

            fp.flush()

    def redict(self, targets = None, mode = 'a', 
            redirect_traceback = True, output_traceback = 'stdout', 
            display_type_traceback = 'colored', remove_esc_sequence = 'file'):
        return self.redirect(targets = targets, mode = mode,
                            redirect_traceback = redirect_traceback, output_traceback = output_traceback, 
                            display_type_traceback = display_type_traceback, remove_esc_sequence = remove_esc_sequence)

    def redirect(self, heading = None, targets = None, mode = 'a', 
            redirect_traceback = True, output_traceback = 'stdout', 
            display_type_traceback = 'colored', remove_esc_sequence = 'file',
            pause_at_terminate = True, ignore_error = True):
        self.redirect_traceback     = redirect_traceback
        self.display_type_traceback = display_type_traceback
        self.remove_esc_sequence    = remove_esc_sequence

        if targets is None:
            for s in self.redirects:
                if s == 'stdout':
                    pass
                elif type(s) is not str:
                    try:
                        s.close()
                    except:
                        pass
            self.redirects = None

        if type(targets) is str:
            targets = [targets]

        if heading is not None:
            print(heading)

        builtins.print = self.print_redirect
        for s in targets:
#            self.print_warning("s=", s)
            if s is None:
                continue
            if s == 'stdout':
                self.redirects.append(s)
            else:
#                if not os.path.isfile(s):
#                    if ignore_error:
#                        self.print_warning(f"\nWarning: [{s}] is not a file. Ignore")
#                        continue
#                    else:
#                        self.print_warning(f"\nError: [{s}] is not a file. Terminate")
#                        self.terminate("")

                try:
                    fp = open(s, mode)
                except:
                    if ignore_error:
                        self.print_warning(f"\nWarning: Can not open [{s}]. Ignore")
                        continue
                    else:
                        self.print_warning(f"\nError: Can not open [{s}]. Terminate")
                        self.terminate("")

                self.redirects.append(fp)

        if redirect_traceback:
            self.redirect_exception(output = output_traceback, display_type = display_type_traceback, pause_at_terminate = pause_at_terminate)

    def replace_path(self, path = None, template = None, ext_dict = {}):
        if template is None:
            template = os.path.join("{dirname}", "{filebody}-out.txt")
        if path is None:
            path = self.script_fullpath

        if type(template) is list:
            p = []
            for s in template:
                s2 = replace_path(path, s, ext_dict)
                p.append(s2)
            return os.path.join(p[0], *p[1:])
        else:
            return replace_path(path, template, ext_dict)

    def get_inifile_dir(self, use_user_inifile = False,
                        inifile_dirs = ['ini'], create_inidir = False,
                        defval = None):
        if not use_user_inifile:
            return self.script_dir

        for dir in inifile_dirs:
            if dir is None:
                continue

            path = os.path.join(self.script_dir, dir)
            path = os.path.abspath(path)
            if create_inidir or os.path.isdir(path):
                return path

        return defval

    def get_inifile_path(self, inidir, use_user_inifile = False):
        dirname, basename, filebody, ext = split_file_path(self.script_fullpath)
        if not use_user_inifile:
            inifile = os.path.join(inidir, f'{filebody}.ini')
        else:
            inifile = os.path.join(inidir, f'{filebody}_{self.user_nospace}.ini')
        
        inifile = os.path.abspath(inifile)
        
        return inifile

#pathが入力ファイル(infile_ext)か設定ファイル(config_ext)であるかによって、infileとconfig_pathを返す
#pathが設定ファイルの場合、infileとinfile_storedはNone
#pathが入力φの場合、config_pathを作る。拡張子はconfig_extの1つめ
    def analyze_infile(app, path, infile_ext = [], config_ext = [], print_level = 1):
        ext = os.path.splitext(path)

        if len(ext) >= 2: ext = ext[1].lower()
        infile_ext = [s.lower() for s in infile_ext]
        config_ext = [s.lower() for s in config_ext]

        print()
        print(f"Check infile parameter:")
        if len(infile_ext) > 0 and ext in config_ext:
            config_path = path
            if print_level:
                print(f"  Parameter file [{config_path}] is given.")
                print(f"  Input file will be red from [{config_path}].")
            infile_stored = None
            infile = None
        else:
            infile = path
            if len(config_ext) >= 1:
                cext = config_ext[0]
            else:
                cext = ''
            config_path = app.replace_path(infile, template = ["{dirname}", "{filebody}" + cext])
            if print_level:
                print(f"  Input file [{infile}] is given.")
                print(f"  Paramter file is set to [{config_path}].")
            infile_stored = infile

        return infile, config_path, infile_stored

    def read_app_config(self, path = None, print_level = 0):
        if path is None:
            path = self.replace_path(None, template = ["{dirname}", "{filebody}.cfg"])

        if print_level:
            print()
            print(f"Read app configuration from [{path}]")
        self.config.read_parameters(path)
        self.appconfig_path = path

        return self.appconfig_path, self.config

    def get_args(self):
        argv = sys.argv
        return argv, len(argv)

    def get_argv(self):
        return sys.argv, len(sys.argv)

    def get_params(self):
        return self.params

    def get_param_dict(self):
        return self.params.get_param_dict()

    def get_param_dic(self):
        return self.params.__dict__

    def getarg(self, key, defval = None):
        if type(key) is int:
            try:
                return self.args_idx[key]
            except:
                return defval
        else:
            try:
                return self.args_opt[key]
            except:
                return defval

    def getarg(self, varname, defval = None, vartype = 'str'):
        iposarg = 0
        for s in sys.argv[1:]:
            if '=' not in s:
                if type(varname) is int and varname == iposarg:
                    return s
                else:
                    iposarg += 1

            _aa = s.split('=')
            if _aa[0] == varname:
                if len(_aa) == 1:
                    return 1
                else:
                    return pconv_by_type(_aa[1], vartype, defval = defval, strict = True)

        return defval

    def check_arg(self, varname, defval = None, vartype = 'str'):
        allowed_keys = self.args_opt_inf.keys()
        if varname in allowed_keys:
            d = self.args_opt_inf[varname]
            defval = d.get("defval", defval)
        
        iposarg = 0
        for s in sys.argv[1:]:
            if len(s) >= 3 and s[:2] == '--':
                pass
            elif '=' not in s:
                if type(varname) is int and varname == iposarg:
                    return s
                else:
                    iposarg += 1
                continue

            _aa = s.split('=')
            if _aa[0] == varname:
                if len(_aa) == 1:
                    return False
                else:
                    ret = pconv_by_type(_aa[1], vartype, defval = defval, strict = True)
                    return ret
        
        return defval

    def make_config_files(self, arg_config_file = None, fit_config_file = None):
#引数定義ファイル
        if arg_config_file is None:
            arg_config_file = self.replace_path(None, template = ["{dirname}", "{filebody}_arg_config.xlsx"])
#フィッティング変数定義ファイル
        if fit_config_file is None:
            fit_config_file = self.replace_path(None, template = ["{dirname}", "{filebody}_fit_config.xlsx"])


        app = tkApplication()

        app.varname    = ["conc",     "qf1",    "nta1", "wta1",    "nga1", "wga1", "ega1",    "nga2", "wga2", "ega2",    "nga3", "wga3", "ega3",    "nga4", "wga4", "ega4",   "N_NI0"]
        app.unit       = [    "",        "", "cm-3/eV",   "eV", "cm-3/eV",   "eV",   "eV", "cm-3/eV",   "eV",   "eV", "cm-3/eV",   "eV",   "eV", "cm-3/eV",   "eV",   "eV",    "cm-3"]
        app.pk_scale   = [ "log",        "",     "log",     "",     "log",     "",     "",     "log",     "",     "",     "log",     "",     "",     "log",     "",     "",        ""]
        app.optid      = [     1,         1,         0,      0,         0,      1,      1,         1,      1,      1,         1,      1,      1,         1,      1,      1,         1]
        app.x0         = [ 1.0e17, -5.08362,    0.0e20,  0.020,    3.0e16,   0.00,    0.0,    5.8e15,   0.41,   0.20,    5.8e15,   0.21,   0.30,    5.8e15,   0.21,   0.50, 6.394e+19]
        app.dx         = [ 5.0e17,      5.0,    2.0e20, 1.0e-5,    1.0e19,   0.10,   0.00,    2.0e17,   0.10,   0.05,    2.0e16,   0.20,   0.30,    2.0e17,   0.10,   0.05,    1.0e19]
        app.kmin       = [ 1.0e13,  -1.0e23,    0.0e10, 1.0e-6,    0.0e10,  0.001,  -0.20,    0.0e10,   0.01,   0.01,    0.0e10,   0.01,   0.20,    0.0e10,   0.01,   0.40,    4.0e19]
        app.kmax       = [ 1.0e20,   1.0e23,    1.0e23,    2.0,    1.0e23,    2.0,    2.0,    1.0e23,    2.0,    0.3,    1.0e23,    2.0,    2.0,    1.0e23,    2.0,    2.0,    8.0e19]
        app.kpenalty   = [    1.0,      1.0,       1.0,    1.0,       1.0,    1.0,    1.0,       1.0,    1.0,    1.0,       1.0,    1.0,    1.0,       1.0,    1.0,    1.0,       1.0]


        app.add_argument(opt = None, type = "str", var_name = 'file1',  opt_str = "file1 (for mode=plot: [DOS|IV|history|fit|DLL]", desc = 'additional file #1', defval = '')
        app.add_argument(opt = None, type = "str", var_name = 'file2',  opt_str = "file2", desc = 'additional file #2', defval = '')
        app.add_argument(opt = None, type = "str", var_name = 'file3',  opt_str = "file3", desc = 'additional file #3', defval = '')
        app.add_argument(opt = None, type = "str", var_name = 'file4',  opt_str = "file4", desc = 'additional file #4', defval = '')
        app.add_argument(opt = None, type = "str", var_name = 'file5',  opt_str = "file5", desc = 'additional file #5', defval = '')

        app.add_argument(opt = "--help", type = "str", defval = 0, optional = True)
        app.add_argument(opt = "--mode", type = "str", var_name = 'mode',  opt_str = "--mode=[init|update|sim|scan|fit|plot]", desc = 'task mode',
                     defval = 'fit', optional = True)

        app.add_argument(opt = '--infile', type = "str", var_name = 'infile', opt_str = "--infile=path",  desc = 'reference (observed IV) Excel file', 
                     defval = 'Vd_0.1V_L8_W20_P-230119-02-3_303K.xlsx', optional = True)
        app.add_argument(opt = "--atlasfile", type = "str", var_name = 'atlasfile', opt_str = "--atlasfile=path",  desc = 'ATLAS input file', 
                     defval = '303K_Vd01.in', optional = True)
        app.add_argument(opt = '--templatepath', type = "str", var_name = 'templatepath', opt_str = "--templatepath=path",  desc = 'template file path (*.in)', 
                     defval = './template', optional = True)
        app.add_argument(opt = "--outfile", type = "str", var_name = 'outfile', opt_str = "--outfile=path",  desc = 'output Excel file', 
                     defval = '303K_Vd01.out', optional = True)

        app.add_argument(opt = "--Cfile", type = "str", var_name = 'Cfile', opt_str = "--Cfile=path",  desc = 'ATLAS C-Interpreter file', 
                     defval = 'mobility_polyOS240507_Tmin250_er14_log.c', optional = True)

        app.add_argument(opt = "--defectaccfile", type = "str", var_name = 'defectaccfile', opt_str = "--defectaccfile=path",  desc = 'ATLAS acceptor-type DOS file', 
                     defval = 'defacc.out', optional = True)
        app.add_argument(opt = "--defectdonfile", type = "str", var_name = 'defectdonfile', opt_str = "--defectdonfile=path",  desc = 'ATLAS donor-type DOS file', 
                     defval = 'defdon.out', optional = True)
        app.add_argument(opt = "--atlaslogfile", type = "str", var_name = 'atlaslogfile', opt_str = "--atlaslogfile=path",  desc = 'ATLAS output (.log) file', 
                     defval = 'idvg_vd001.log', optional = True)
        app.add_argument(opt = "--calfile", type = "str", var_name = 'calfile', opt_str = "--calfile=path",  desc = 'ATLAS output cal (.dat) file', 
                     defval = "idvg_calc.dat", optional = True)

        app.add_argument(opt = '--IVfile', type = "str", var_name = 'IVfile', opt_str = "--IVfile=path",  desc = 'observed IV output Excel file', 
                     defval = 'IVobs.xlsx', optional = True)
        app.add_argument(opt = "--fitfile", type = "str", var_name = 'fitfile', opt_str = "--fitfile=path",  desc = 'Fiting result file', 
                     defval = 'IVfit.xlsx', optional = True)


        app.add_argument(opt = "--target_var", type = "str",   defval = 'conc')
        app.add_argument(opt = "--x0",         type = "float", defval = 0.0)
        app.add_argument(opt = "--x1",         type = "float", defval = 1.0)
        app.add_argument(opt = "--nx",         type = "int",   defval = 5)

        app.add_argument(opt = "--Eg", type = "float", desc = 'Band gap', defval = 2.8)
        app.add_argument(opt = "--T", type = "float", desc = 'Temperature', defval = 303.0)

        app.add_argument(opt = "--ix_plot", type = "int|str", defval = 0, desc = 'index of x_data to plot as x-axis', optional = True)
        app.add_argument(opt = "--x_scale", type = "str", defval = "", desc = 'x scale to plot [blank|log]', optional = True)
        app.add_argument(opt = "--fhistory", type = "int", var_name = 'fhistory',  opt_str = "--fhisotry=[0|1]", desc = 'flag to save hisotry file',
                     defval = 1, optional = True)
        app.add_argument(opt = "--fmax_record", type = "double", var_name = 'fmax_record',  opt_str = "--fmax_record=val", desc = 'max fmin value to record to file',
                     defval = 1.0, optional = True)
        app.add_argument(opt = "--ffitfiles", type = "int", var_name = 'ffitfiles',  opt_str = "--ffitfiles=[0|1]", desc = 'flag to save fit files',
                     defval = 1, optional = True)

        app.add_argument(opt = "--fplot", type = "int", var_name = 'fplot',  opt_str = "--fplot=[0|1]", desc = 'flag to plot graph',
                     defval = 1, optional = True)

        for i, varname in enumerate(app.varname):
            app.add_argument(opt = f"--{varname}", type = "float", defval = app.x0[i])
        app.add_argument(opt = f"--y_convert", type = "str", defval = "log")
        app.add_argument(opt = f"--f_max_plot", type = "float", defval = 10.0)

        app.add_argument(opt = "--method", type = "str", var_name = 'method',  opt_str = "--method=[nelder-mead]", desc = 'optimization algorism',
                     defval = 'nelder-mead', optional = True)
        app.add_argument(opt = "--jac", type = "str", var_name = 'jac',  opt_str = "--jac=[3-points|2-points|func]", desc = 'First differential',
                     defval = '3-points', optional = True)
        app.add_argument(opt = "--nmaxiter", type = "int", var_name = 'nmaxiter',  opt_str = "--nmaxiter=int(>1)", desc = 'maximum interation number for optimization',
                     defval = 1000, optional = True)
        app.add_argument(opt = "--nmaxcall", type = "int", var_name = 'nmaxcall',  opt_str = "--nmaxcall=int(>1)", desc = 'maximum number for minimize_func calls',
                     defval = 10000, optional = True)
        app.add_argument(opt = "--tol", type = "float", var_name = 'tol',  opt_str = "--tol=val", desc = 'eps for optimization',
                     defval = 1.0e-5, optional = True)
    
        app.add_argument(opt = "--print_interval", type = "int", var_name = 'print_interval',  opt_str = "--print_interval=val", desc = 'print interval',
                     defval = 5, optional = True)
        app.add_argument(opt = "--plot_interval", type = "int", var_name = 'plot_interval',  opt_str = "--plot_interval=val", desc = 'plot interval',
                     defval = 5, optional = True)

#=============================
# Graph configuration
#=============================
        app.add_argument(opt = "--fontsize",            type = "int", defval = 16, optional = True)
        app.add_argument(opt = "--legend_fontsize",     type = "int", defval = 12, optional = True)
        app.add_argument(opt = "--graphupdateinterval", type = "int", defval = 10, optional = True)
        app.add_argument(opt = "--use_tkplt",           type = "int", defval = 0, optional = True)

        print(f"Save argument configuration to [{arg_config_file}]")
        app.save_arg_config(arg_config_file)
        print(f"Save fitting configuration to [{fit_config_file}]")
        vars = tkParams()
        vars.varname = app.varname
        vars.unit = app.unit
        vars.pk_scale = app.pk_scale
        vars.optid = app.optid
        vars.x0 = app.x0
        vars.dx = app.dx
        vars.kmin = app.kmin
        vars.kmax = app.kmax
        vars.kpenalty = app.kpenalty
        save_fit_config(vars, fit_config_file)

        return True

    def read_arg_config_from_file(self, path, labels = ["argtype", "var_name", "opt", "opt_str", "desc", "type", "defval", "optional"]):
        if not os.path.exists(path): return False

        idx_argtype  = labels.index("argtype")
        idx_opt      = labels.index("opt")
        idx_varname  = labels.index("var_name")
        idx_optstr   = labels.index("opt_str")
        idx_desc     = labels.index("desc")
        idx_type     = labels.index("type")
        idx_defval   = labels.index("defval")
        idx_optional = labels.index("optional")

        pos_args = self.args_idx_inf
        key_args = self.args_opt_inf

        wb = openpyxl.load_workbook(path)

        sheet = wb.active
        _labels = [cell.value for cell in sheet[1]]
        for row in sheet.iter_rows(min_row = 2):
            values = []
            for r in row:
                v = r.value 
                t = type(v)
                if v is None: values.append('')
                elif t is str:
                    if   v == 'None': values.append('')
                    elif v == 'True': values.append(1)
                    elif v == 'Fales': values.append(0)
                    else: values.append(v)
                else:
                    values.append(v)

#            print("row=", values)
            arg_type = values[idx_argtype]
            if arg_type not in ["keyword", "positional"]: continue

            opt      = values[idx_opt]
            varname  = values[idx_varname]
            optstr   = values[idx_optstr]
            desc     = values[idx_desc]
            _type    = values[idx_type]
            defval   = values[idx_defval]
            optional = values[idx_optional]
            d = {"var_name": varname, "opt": opt, "opt_str": optstr, "desc": desc, 
                 "type": _type, "defval": defval, "optional": optional}
            if values[idx_argtype] == 'positional':
                pos_args.append(d)
            else:
                key_args[opt] = d

        wb.close()
#        print("key_args=", key_args["--fmax_record"])
        
        return True
        
    def save_arg_config(self, outfile, labels = ["argtype", "var_name", "opt", "opt_str", "desc", "type", "defval", "optional"]):
        data_list = []
        for inf in self.args_idx_inf:
            args = ["positional"]
            for idx, l in enumerate(labels):
                if idx == 0: continue
                v = val2str(inf[l])
                args.append(v)
            data_list.append(args)

        for key, inf in self.args_opt_inf.items():
            key = key.lstrip('-')
            args = ["keyword"]
            for idx, l in enumerate(labels):
                if idx == 0: continue
                v = val2str(inf[l])
                args.append(v)
            data_list.append(args)

        data_list = list(map(list, zip(*data_list)))
        tkVariousData().save(outfile, data_list = data_list, labels = labels)
    
    def add_argument(self, opt = None, opt_str = None, var_name = None, desc = None, type = 'str', defval = None, optional = True):
#        if opt is None:
#            opt = "--" + var_name
        if opt is not None and var_name is None and opt[0:2] == '--':
            var_name = opt.lstrip('-')
        if opt_str is None:
            if opt is None:
                opt_str = f"#{len(self.args_idx_inf)+1} arg: {var_name}: val"
            else:
                opt_str = f"--{var_name}=val"
        if desc is None:
            desc = var_name

        inf_idx = self.args_idx_inf
        inf_opt = self.args_opt_inf
        if opt is None:
# keywordなし引数
            i = len(inf_idx)
            inf_idx.append( { "var_name": var_name, "opt": i, "opt_str": opt_str, "desc": desc, 
                              "type": type, "defval": defval, "optional": optional } )
        else:
# keywordあり引数
            inf_opt[opt] = { "var_name": var_name, "opt": opt, "opt_str": opt_str, "desc": desc, 
                              "type": type, "defval": defval, "optional": optional }

        return inf_opt, inf_idx

    def check_help(self, pause = True):
        argv = sys.argv
        for s in argv:
            _aa = s.split("=", 1)
            if len(_aa) == 0:
                _s = s
            else:
                _s = _aa[0]

            s = s.lower()
            if s == '--help' or s == '-?' or s == '/?' or s == '?' or s == 'help':
                print()
                print("#==============================================")
                print(f"Found the argument [{s}] for help:")
                print("#==============================================")
                self.terminate(usage = 'built-in', pause = pause)
    
    def read_args(self, vars = None, check_allowed_args = False, apply_default = True, pause = True):
        self.check_help(pause = pause)

        if vars is not None and type(vars) is not dict:
            vars = vars.__dict__
        argv = sys.argv
        args_idx = []
        args_opt = {}

#        print(f"{vars=}")
        if vars is None:
            args_vars = {}
        else:
            args_vars = vars
                    
        allowed_keys = self.args_opt_inf.keys()
        i_idx_arg = 0
#        print("argv=", argv)
        given_args = {}
        for i in range(1, len(argv)):
            arg = del_quote(argv[i])
#-/--で始まるが、=以降が無いとき
            m = re.match(r'^\s*\-{1,2}\w([\-\.\w]*?)[^=]*$', arg)
            if m:
                arg = arg + "=1"
                print()
                print(f"Warning in tkapplication.read_args(): Argument [{argv[i]}] does not have a given value.")
                print(f"   Assume the value is 1 [{arg}]")
                print()
            
#-/--で始ろ、=以降を与えられているとき
            m = re.match(r'^\s*(\-{1,2}\w[\-\.\w]*?)\s*=\s*(.*)\s*$', arg)
#            m = re.match(r'\s*(\S.*)\s*=\s*(.*)\s*$', arg)
            if m:
# keyword引数
                opt = m.groups()[0]
                val = m.groups()[1]
#                print("opt=", opt, val)
#                print(f"i={i} keyword arg: {opt}={val}")
                if check_allowed_args and opt not in allowed_keys:
                    return None, 1, f"Error in tkapplicaiton.read_args(): Invalid argument [{opt}]"
#                    print(f"Error in tkapplicaiton.read_args(): Invalid argument [{opt}]")
#                    exit()
                else:
                    var_name = opt.lstrip("-")
                    if opt not in allowed_keys:
                        print()
                        print("====================================================================")
                        print(f"Warning in tkapplication.read_args(): Key {opt} is not defined.")
                        print(f"  {val} is added to vars[{var_name}].")
                        print("====================================================================")
                        print()
                    args_vars[var_name] = pconv(val)

                if opt in allowed_keys:
                    var_type = self.args_opt_inf[opt].get("type", None)
                    if var_type is not None:
                        args_opt[opt] = pconv_by_type(val, var_type, defval = None, strict = True)
                        if args_opt[opt] is None:
                            return None, 2, \
                                f"Error: Argument [{opt}] = [{val}] is not compatible with the required type [{var_type}]"
                        given_args[opt] = args_opt[opt]

# attributeに設定
                    var_name = self.args_opt_inf[opt]["var_name"]
                    if var_name is not None:
#                        print(f"i={i} arg={arg} set var_name={var_name} args_opt={args_opt[opt]}")
                        args_vars[var_name] = args_opt[opt]
                else:
                    args_opt[opt] = val
                    given_args[opt] = val
            else:
# keywordなし引数
#                print(f"i={i} positional arg: {arg}")
                if i_idx_arg >= len(self.args_idx_inf):
# keywordなし引数の数が定義数より大きい場合
                    idx = len(self.args_idx_inf) - 1
                else:
# keywordなし引数の数が定義数以下の場合
                    idx = i_idx_arg

                var_type = self.args_idx_inf[idx].get("type", None)
                if var_type is None or var_type == 'list':
                    val = arg
                else:
                    val = pconv_by_type(arg, var_type, defval = None, strict = True)
                    if val is None:
                        return None, 3, \
                               f"Error: #{i_idx_arg} positional argument = [{arg}] is not compatible with the required type [{var_type}]"

                given_args[str(idx)] = val
                args_idx.append(val)
                i_idx_arg += 1

                var_name = self.args_idx_inf[idx]["var_name"]
                if var_name is not None:
#                    print(f"i={i} arg={arg}: set var_name={var_name} to [{val}]")
                    if var_type == 'list':
                        if args_vars.get(var_name, None) is None:
                            args_vars[var_name] = [val]
                        else:
                            args_vars[var_name].append(val)
                    else:
                        args_vars[var_name] = val

        if apply_default:
# 引数が与えられていない変数にデフォルト値を設定
            for key in allowed_keys:
                v = self.args_opt_inf[key]
                var = v["var_name"]
                if args_opt.get(key, None) is None:
#                    print(f"default {var=} {v['defval']=}")
                    args_opt[var]  = v["defval"]
                    args_vars[var] = v["defval"]

                    if not v["optional"]:
                        return None, 4, f"Error: Argument [{key}] is required but not given"
#                print("")
#                print(f"Error: Argument [{key}] is required but not given")
#                exit()

# positional arguments
            nposargs = len(args_idx)
            for i in range(len(self.args_idx_inf)):
                v = self.args_idx_inf[i]
#                print(f"positional args: i={i} v={v}")
                if i >= nposargs:
                    var = self.args_idx_inf[i]["var_name"]
                    args_vars[var] = v["defval"]
                    if not v["optional"]:
                        return None, 5, f"Error: #{i} positional argument is required but not given"
#                print("")
#                print(f"Error: #{i} positional argument is required but not given")
#                exit()
        
        self.given_args = given_args
        self.args_opt   = args_opt
        self.args_idx   = args_idx
        self.args_vars  = args_vars

        return args_opt, args_idx, args_vars
    
    def force_given_args(self, cparams):
        if not hasattr(self, "args_idx"): return 

        nidxargs = len(self.args_idx)
        for i, inf in enumerate(self.args_idx_inf):
            opt = inf.get("opt", None)
            if opt is None: continue
            if nidxargs <= opt: break

#            print("opt=", opt)
            if type(opt) == int:
                varname = inf.get("var_name", None)
                vartype = inf.get("type", None)
                defval  = inf.get("defval", None)
                val = self.args_idx[opt]
                val = pconv_by_type(val, vartype, defval = defval, strict = True)
                setattr(cparams, varname, val)
        
#        print("given args:", self.given_args)
        for key, val in self.given_args.items():
            inf = self.args_opt_inf.get(key, None)
            if inf is None: continue

            varname = inf.get("var_name", None)
            vartype = inf.get("type", None)
            defval  = inf.get("defval", None)
#            key = key.lstrip('-')
            val = pconv_by_type(val, vartype, defval = defval, strict = True)
            setattr(cparams, varname, val)

    def update_vars(self, vars = None, usage = None, check_allowed_args = True, apply_default = True):
        if vars is None:
            vars = self.cparams
            
        if usage is None: usage = self.usage

        args_opt, args_idx, args_vars = self.read_args(vars = vars, check_allowed_args = check_allowed_args, apply_default = apply_default)
        if args_opt is None:
            self.error_no = args_idx
            self.error_message = args_vars
            self.terminate(message = "\n",
                   usage = usage,
                   post_message = ""
                       +  "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
                       + f"!  {self.error_message}\n"
                       +  "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
                   )

        return args_opt, args_idx, args_vars


    def set_usage(self, usage_str = '', suppress_usage = ''):
        self.usage_str      = usage_str
        self.suppress_usage = suppress_usage

    def usage(self, *args):
        if 'all' in self.suppress_usage:
            return

        if self.usage_str is not None and self.usage_str != '' and 'usage' not in self.suppress_usage:
            if self.args_vars:
                s_conv = convert_by_vars(self.usage_str, self.args_vars, prefix = '', bra = r'\{', ket = r'\}')
            else:
                s_conv = self.usage_str

#            print("293")
            for s in s_conv.split('\n'):
                if not re.match(r'\s*[\"\'].*[\"\']', s):
                    s = '"' + s + '"'
                cmd = 'print({})'.format(s.rstrip())
#                print("cmd=", cmd)
                eval(cmd)
            return

        inf_opt = self.args_opt_inf
        keys_opt = inf_opt.keys()
        inf_idx = self.args_idx_inf
#        print("inf_opt=", inf_opt)
#        print("inf_idx=", inf_idx)
        if len(keys_opt) > 0 or len(inf_idx) > 0:
            usage = f'Usage: python {sys.argv[0]} '
            for key in keys_opt:
                v = inf_opt[key]
                if v['optional']:
                    usage += f"[{v['opt_str']}] "
                else:
                    usage += f"{v['opt_str']} "
            for i in range(len(inf_idx)):
                v = inf_idx[i]
                if v['optional']:
                    usage += f"[{v['opt_str']}] "
                else:
                    usage += f"{v['opt_str']} "

            if 'usage' not in self.suppress_usage:
                print(usage)

            if 'options' not in self.suppress_usage:
                print(f"  Options:")
                for key in keys_opt:
                    v = conv_float(inf_opt[key])
                    self.print_warning(f"    {v['opt_str']} (def={v['defval']}) (type={v['type']})")
                    if v['desc'] != '':
                        print(f"        {v['desc']}")
                for i in range(len(inf_idx)):
                    v = conv_float(inf_idx[i])
                    self.print_warning(f"    {v['opt_str']} (def={v['defval']}) (type={v['type']})")
#                    self.print_warning(f"    #{i} {v['opt_str']} (def={v['defval']}) (type={v['type']})")
                    if v['desc'] != '':
                        print(f"        {v['desc']}")

            return

    def terminate(self, message = None, post_message = None, input_text = "Press ENTER to terminate>>\n", usage = None, pause = False):
        if self.save_time: self.print_time(label = "startup", label2 = "end")

        if message is not None:
            self.print_warning("")
            self.print_warning(message)

        if usage:
            if usage == 'built-in' or  usage == 'build-in':
                self.usage()
            else:
                self.print_warning("")
                usage(self)

        if post_message:
            self.print_warning("")
            self.print_warning(post_message)

        if pause:
            print("")
            input(input_text)

        if message is not None or post_message is not None or usage is not None or pause:
            print("")

        exit()

    def print_time(self, label = None, label2 = None):
        if label is None or label not in self.time_stamps.keys():
            tprec = time.perf_counter()
            tnow = datetime.now()
            self.time_stamps[label] = [tprec, tnow.strftime("%y/%m/%d %H:%M:%S")]

        print(f"time at {label}:", self.time_stamps[label][1])
        
        if label2 is not None:
            if label2 not in self.time_stamps.keys():
                tprec = time.perf_counter()
                tnow = datetime.now()
                self.time_stamps[label2] = [tprec, tnow.strftime("%y/%m/%d %H:%M:%S")]

            print(f"time at {label2}:", self.time_stamps[label2][1])
            print(f"Elapsed time from {label} to {label2}: {self.time_stamps[label2][0] - self.time_stamps[label][0]:10.6g} sec")
        
    def print_title(app, message):
        print()
        print( "#========================================================")
        print(message)
        print( "#========================================================")
    
    def write_resource(app, path, mode):
        with open(f, 'w') as fp:
            for key in app.phrases.keys():
                try:
                    val = app.phrases[key][lang]
                    fp.write(f'{key}:{val}' + '\n')
                except:
                    pass
        return True

    def read_resource(app, path, lang, phrases = None, sep = '='):
        if phrases is None:
            phrases = {}

        fp = open_chardet(path, 'r', def_encoding = 'shift_jis')
#        fp = open(path, 'r')
        if not fp:
            return phrases

        while 1:
            line = fp.readline()
            if not line:
                break

#            m = re.match(rf'\s*(\S.*)\s*=\s*(.*)\s*\n$', line, flags = re.MULTILINE | re.DOTALL)
            m = re.match(rf'\s*(\S.*)\s*{sep}\s*(.*)\s*\n$', line, flags = re.MULTILINE | re.DOTALL)
            if not m:
                continue
            
            var = m.groups()[0]
            val = m.groups()[1]
            if phrases.get(var, None) is None:
                phrases[var] = {}
            
            phrases[var][lang] = val
#            print(f"{var=} {val=}")

        fp.close()
#        print(f"{phrases=}")
#        exit()
        
        return phrases

    def p(self, str, defval = None, phrases = None):
        if phrases is None:
            phrases = self.phrases

        if str is None or str == '':
            return ''

#        print("")
#        print(f"{str=}")
        s = phrases.get(str, defval)
#        print(f"{s=}")
        if s is None:
            if defval is not None:
                print(f" 336 return defval={defval}")
                return defval
 
#            print(f" return {str=}")
            return str

        s = s.get(self.config.lang, defval)
#        print(f"{s=}")
        if s is None:
            if defval is not None:
#                print(f" 346 return defval={defval}")
                return defval
 
#            print(f" return {str=}")
            return str

#        print(f"last: return {s=}")
        return s

    def print(self, *args, **kwargs):
        print(*args, **kwargs)

    def print_attributes(self, print_header = 1):
         keys = self.__dict__.keys()
         if print_header:
             print("attributes in {}".format(self.ClassPath()))
         for key in keys:
            print("  {}: {}".format(key, self.__dict__[key]))

    def print_attributes_warning(self, print_header = 1):
         keys = self.__dict__.keys()
         if print_header:
             self.print_warning("attributes in {}".format(self.ClassPath()))
         for key in keys:
            self.print_warning("  {}: {}".format(key, self.__dict__[key]))

    def print_dict(self, d, header = None):
        print("")
        if header is not None:
            print(header)
        
        if d is None:
            print("  Null dict")
            return

        for key in d.keys():
            print(f"{key}: {d[key]}")

    def print_dict_warning(self, d, header = None):
        print("")
        if header is not None:
            self.print_warning(header)
        
        if d is None:
            self.print_warning("  Null dict")
            return

        for key in d.keys():
            self.print_warning(f"{key}: {d[key]}")

    def load_module(self, module_name, target = '', plugin_dir = '', desc = '', ret_type = 'module', is_print = False):
        if plugin_dir != '' and plugin_dir not in sys.path:
            sys.path.append(plugin_dir)

#        module = import_module(module_name)
        try:
            module = import_module(module_name)
        except Exception as e:
            print(f"Warning in tkapplication.load_module(): Error due to {e}")
            traceback.print_exc()
            module = None

        if module is None:
            if is_print:
                print(f"Error in tkapplication.load_module(): Failed to load module [{module_name}]")
            return None
        else:
            setattr(module, 'name', module_name)
            setattr(module, 'target', target)
            setattr(module, 'desc', desc)
            tkmodule = tkModule(name = module_name, module = module)
            self.modules.append({"module": module, "tkModule": tkmodule, "module_name": module_name, "desc": desc})

        if ret_type == 'tkmodule':
            return tkmodule

        if ret_type == 'module':
            return module

        return module_name

    def load_modules(self, plugin_dir = '', fmask = '*.py', target = "read_data", ret_type = 'module', sort = False, is_print = True):
        search_path = os.path.join(plugin_dir, fmask)
        if is_print:
            print(f"Search modules: {plugin_dir}/{fmask}")
#        print(f"tkapplication.load_modules(): plugin_dir: {plugin_dir}")
#        print(f"tkapplication.load_modules(): fmask: {fmask}")
# Search plug-in files

        files = glob.glob(search_path)
        if sort:
            files.sort()
        module_names = []
        for path in files:
            dirname, basename, filebody, ext = split_file_path(path)
            module_names.append(filebody)

        if plugin_dir != '' and plugin_dir not in sys.path:
            sys.path.append(plugin_dir)

        modules = []
        module_names2 = []
        for i in range(len(module_names)):
            m = module_names[i]
#plugin_dirはsys.path()に設定しているので、load_module()には '' を渡し、重ねてsys.path()に登録しないようにする
            module = self.load_module(module_name = m, plugin_dir = '', target = target, desc = m, ret_type = ret_type, is_print = False)
            if module:
                modules.append(module)
                module_names2.append(module.name)
                if is_print:
                    print(f"  {m}: loaded")
            else:
                if is_print:
                    print(f"Warning in tkapplication.load_modules(): Invalid module file [{m}.py]")

        return module_names2, modules

    def call(self, module_name, func_name, *args, **kwargs):
        if type(module_name) is not str:
            module_name = module_name.name
#            m = re.search(r"module\s+'(.*?)'", str(module_name))
#            if m:
#                module_name = m.groups()[0]

        ret = None
        for minf in self.modules:
            if minf["module_name"] == module_name:
#                print("module:", minf["module_name"], minf["module"])
                module = minf["module"]
#                print("module names:", module.__dict__.keys())
                
                if func_name not in module.__dict__.keys():
#                    return None
                    pass
                else:
                    ret = module.__dict__[func_name](*args, **kwargs)
                    return ret

#        print(f"call globals.{func_name}")
        if func_name not in self.globals.keys():
            self.terminate(f"Error in tkapplication.call(): Can not find the function [{func_name}] in self.globals\n")
#            print("")
#            print(f"Error in tkapplication.call(): Can not find the function [{func_name}] in self.globals\n")
#            return None
        else:
            ret = self.globals[func_name](*args)

        return ret

    def call_all(self, func_name, *args):
        ncall = 0
        ret = None
        for minf in self.modules:
#            print("module:", minf["module_name"], minf["module"])
            if minf["module"].__dict__.get(func_name, None) is not None:
                ret = minf["module"].__dict__[func_name](*args)
                ncall += 1

        if ncall == 0:
#            print(f"call globals.{func_name}")
            ret = self.globals[func_name](*args)

        return ret

    def read_parameters(self, path, merge_params = False, AddSection = 0):
        ini = tkIniFile()
        inf = ini.ReadAll(path, AddSection = AddSection)
        if inf is not None and merge_params:
            self.params += inf

        return inf

    def save_parameters(self, path, params = None, section = 'Parameters', otherparams = None):
        if params is None:
            params = self.get_params()

        ini = tkIniFile(path)
        for i in params.keys():
            val = params.get(key, None)
            if val is not None:
                ini.WriteString(section, key, val)

        if otherparams is not None:
            for key in otherparams.keys():
                val = otherparams.get(key, None)
                if val is not None:
                    ini.WriteString('Preferences', key, val)

    def print_parameters(self, params = None):
        if params is None:
            params = self.get_params()

        for key in params.keys():
            val = params.get(key, None)
#            print("val=", val)
            if val is None:
                print(f"  {key:10s}: {'None':14}")
            else:
                if type(val) is float:
                    print(f"  {key:10s}: {val:14.8g}")
                elif type(val) is int or type(val) is str:
                    print(f"  {key:10s}: {val:14}")
                else:
                    print(f"  {key:10s}:", val)


def main():
    print("")
    print("This is library, not runnable")
    print("")


if __name__ == "__main__":
    main()
