diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 73104f2..2292b63 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -12,4 +12,10 @@ API Reference runmanager.remote runmanager.batch_compiler runmanager.globals_diff + runmanager.differ + runmanager.evaluator + runmanager.expander + runmanager.group_manager + runmanager.tokenizer + runmanager.widgets runmanager.__main__ diff --git a/runmanager/__init__.py b/runmanager/__init__.py index c6b4793..28f2f57 100644 --- a/runmanager/__init__.py +++ b/runmanager/__init__.py @@ -11,330 +11,16 @@ # # ##################################################################### -import itertools import os import sys -import random -import time -import subprocess -import types -import threading -import traceback -import datetime -import errno -import json -import tokenize -import io import warnings import labscript_utils.h5_lock -import h5py -import numpy as np - -from labscript_utils.ls_zprocess import ProcessTree, zmq_push_multipart -from labscript_utils.labconfig import LabConfig -import labscript_utils.shot_utils -process_tree = ProcessTree.instance() from .__version__ import __version__ - -def _ensure_str(s): - """convert bytestrings and numpy strings to python strings""" - return s.decode() if isinstance(s, bytes) else str(s) - - -def is_valid_python_identifier(name): - # No whitespace allowed. Do this check here because an actual newline in the source - # is not easily distinguished from a NEWLINE token in the produced tokens, which is - # produced even when there is no newline character in the string. So since we ignore - # NEWLINE later, we must check for it now. - if name != "".join(name.split()): - return False - try: - tokens = list(tokenize.generate_tokens(io.StringIO(name).readline)) - except tokenize.TokenError: - return False - token_types = [ - t[0] for t in tokens if t[0] not in [tokenize.NEWLINE, tokenize.ENDMARKER] - ] - if len(token_types) == 1: - return token_types[0] == tokenize.NAME - return False - - -def is_valid_hdf5_group_name(name): - """Ensure that a string is a valid name for an hdf5 group. - - The names of hdf5 groups may only contain ASCII characters. Furthermore, the - characters "/" and "." are not allowed. - - Args: - name (str): The potential name for an hdf5 group. - - Returns: - bool: Whether or not `name` is a valid name for an hdf5 group. This will - be `True` if it is a valid name or `False` otherwise. - """ - # Ensure only ASCII characters are used. - for char in name: - if ord(char) >= 128: - return False - - # Ensure forbidden ASCII characters are not used. - forbidden_characters = ['.', '/'] - for character in forbidden_characters: - if character in name: - return False - return True - - -class ExpansionError(Exception): - - """An exception class so that error handling code can tell when a - parsing exception was caused by a mismatch with the expansion mode""" - pass - - -class TraceDictionary(dict): - - def __init__(self, *args, **kwargs): - self.trace_data = None - dict.__init__(self, *args, **kwargs) - - def start_trace(self): - self.trace_data = [] - - def __getitem__(self, key): - if self.trace_data is not None: - if key not in self.trace_data: - self.trace_data.append(key) - return dict.__getitem__(self, key) - - def stop_trace(self): - trace_data = self.trace_data - self.trace_data = None - return trace_data - - -def new_globals_file(filename): - """Creates a new globals h5 file. - - Creates a 'globals' group at the top level. - If file does not exist, a new h5 file is created. - """ - with h5py.File(filename, 'w') as f: - f.create_group('globals') - - -def add_expansion_groups(filename): - """backward compatability, for globals files which don't have - expansion groups. Create them if they don't exist. Guess expansion - settings based on datatypes, if possible.""" - # DEPRECATED - # Don't open in write mode unless we have to: - with h5py.File(filename, 'r') as f: - requires_expansion_group = [] - for groupname in f['globals']: - group = f['globals'][groupname] - if 'expansion' not in group: - requires_expansion_group.append(groupname) - if requires_expansion_group: - group_globalslists = [get_globalslist(filename, groupname) for groupname in requires_expansion_group] - with h5py.File(filename, 'a') as f: - for groupname, globalslist in zip(requires_expansion_group, group_globalslists): - group = f['globals'][groupname] - subgroup = group.create_group('expansion') - # Initialise all expansion settings to blank strings: - for name in globalslist: - subgroup.attrs[name] = '' - groups = {group_name: filename for group_name in get_grouplist(filename)} - sequence_globals = get_globals(groups) - evaled_globals, global_hierarchy, expansions = evaluate_globals(sequence_globals, raise_exceptions=False) - for group_name in evaled_globals: - for global_name in evaled_globals[group_name]: - value = evaled_globals[group_name][global_name] - expansion = guess_expansion_type(value) - set_expansion(filename, group_name, global_name, expansion) - - -def get_grouplist(filename): - # For backward compatability, add 'expansion' settings to this - # globals file, if it doesn't contain any. Guess expansion settings - # if possible. - # DEPRECATED - add_expansion_groups(filename) - with h5py.File(filename, 'r') as f: - grouplist = f['globals'] - # File closes after this function call, so have to - # convert the grouplist generator to a list of strings - # before its file gets dereferenced: - return list(grouplist) - - -def new_group(filename, groupname): - if not is_valid_hdf5_group_name(groupname): - raise ValueError( - 'Invalid group name. Group names must contain only ASCII ' - 'characters and cannot include "/" or ".".' - ) - with h5py.File(filename, 'a') as f: - if groupname in f['globals']: - raise Exception('Can\'t create group: target name already exists.') - group = f['globals'].create_group(groupname) - group.create_group('units') - group.create_group('expansion') - - -def copy_group(source_globals_file, source_groupname, dest_globals_file, delete_source_group=False): - """ This function copies the group source_groupname from source_globals_file - to dest_globals_file and renames the new group so that there is no name - collision. If delete_source_group is False the copyied files have - a suffix '_copy'.""" - with h5py.File(source_globals_file, 'a') as source_f: - # check if group exists - if source_groupname not in source_f['globals']: - raise Exception('Can\'t copy there is no group "{}"!'.format(source_groupname)) - - # Are we coping from one file to another? - if dest_globals_file is not None and source_globals_file != dest_globals_file: - dest_f = h5py.File(dest_globals_file, 'a') # yes -> open dest_globals_file - else: - dest_f = source_f # no -> dest files is source file - - # rename Group until there is no name collisions - i = 0 if not delete_source_group else 1 - dest_groupname = source_groupname - while dest_groupname in dest_f['globals']: - dest_groupname = "{}({})".format(dest_groupname, i) if i > 0 else "{}_copy".format(dest_groupname) - i += 1 - - # copy group - dest_f.copy(source_f['globals'][source_groupname], '/globals/%s' % dest_groupname) - - # close opend file - if dest_f != source_f: - dest_f.close() - - return dest_groupname - - -def rename_group(filename, oldgroupname, newgroupname): - if oldgroupname == newgroupname: - # No rename! - return - if not is_valid_hdf5_group_name(newgroupname): - raise ValueError( - 'Invalid group name. Group names must contain only ASCII ' - 'characters and cannot include "/" or ".".' - ) - with h5py.File(filename, 'a') as f: - if newgroupname in f['globals']: - raise Exception('Can\'t rename group: target name already exists.') - f.copy(f['globals'][oldgroupname], '/globals/%s' % newgroupname) - del f['globals'][oldgroupname] - - -def delete_group(filename, groupname): - with h5py.File(filename, 'a') as f: - del f['globals'][groupname] - - -def get_globalslist(filename, groupname): - with h5py.File(filename, 'r') as f: - group = f['globals'][groupname] - # File closes after this function call, so have to convert - # the attrs to a dict before its file gets dereferenced: - return dict(group.attrs) - - -def new_global(filename, groupname, globalname): - if not is_valid_python_identifier(globalname): - raise ValueError('%s is not a valid Python variable name'%globalname) - with h5py.File(filename, 'a') as f: - group = f['globals'][groupname] - if globalname in group.attrs: - raise Exception('Can\'t create global: target name already exists.') - group.attrs[globalname] = '' - f['globals'][groupname]['units'].attrs[globalname] = '' - f['globals'][groupname]['expansion'].attrs[globalname] = '' - - -def rename_global(filename, groupname, oldglobalname, newglobalname): - if oldglobalname == newglobalname: - # No rename! - return - if not is_valid_python_identifier(newglobalname): - raise ValueError('%s is not a valid Python variable name'%newglobalname) - value = get_value(filename, groupname, oldglobalname) - units = get_units(filename, groupname, oldglobalname) - expansion = get_expansion(filename, groupname, oldglobalname) - with h5py.File(filename, 'a') as f: - group = f['globals'][groupname] - if newglobalname in group.attrs: - raise Exception('Can\'t rename global: target name already exists.') - group.attrs[newglobalname] = value - group['units'].attrs[newglobalname] = units - group['expansion'].attrs[newglobalname] = expansion - del group.attrs[oldglobalname] - del group['units'].attrs[oldglobalname] - del group['expansion'].attrs[oldglobalname] - - -def get_value(filename, groupname, globalname): - with h5py.File(filename, 'r') as f: - value = f['globals'][groupname].attrs[globalname] - # Replace numpy strings with python unicode strings. - # DEPRECATED, for backward compat with old files - value = _ensure_str(value) - return value - - -def set_value(filename, groupname, globalname, value): - with h5py.File(filename, 'a') as f: - f['globals'][groupname].attrs[globalname] = value - - -def get_units(filename, groupname, globalname): - with h5py.File(filename, 'r') as f: - value = f['globals'][groupname]['units'].attrs[globalname] - # Replace numpy strings with python unicode strings. - # DEPRECATED, for backward compat with old files - value = _ensure_str(value) - return value - - -def set_units(filename, groupname, globalname, units): - with h5py.File(filename, 'a') as f: - f['globals'][groupname]['units'].attrs[globalname] = units - - -def get_expansion(filename, groupname, globalname): - with h5py.File(filename, 'r') as f: - value = f['globals'][groupname]['expansion'].attrs[globalname] - # Replace numpy strings with python unicode strings. - # DEPRECATED, for backward compat with old files - value = _ensure_str(value) - return value - - -def set_expansion(filename, groupname, globalname, expansion): - with h5py.File(filename, 'a') as f: - f['globals'][groupname]['expansion'].attrs[globalname] = expansion - - -def delete_global(filename, groupname, globalname): - with h5py.File(filename, 'a') as f: - group = f['globals'][groupname] - del group.attrs[globalname] - - -def guess_expansion_type(value): - if isinstance(value, np.ndarray) or isinstance(value, list): - return u'outer' - else: - return u'' - +# Used in BLACS +from runmanager.compiler import compile_labscript_with_globals_files_async def iterator_to_tuple(iterator, max_length=1000000): # We want to prevent infinite length tuples, but we cannot know @@ -349,774 +35,3 @@ def iterator_to_tuple(iterator, max_length=1000000): 'If you really want an iterator longer than %d, ' % max_length + 'please modify runmanager.iterator_to_tuple and increase max_length.') return tuple(temp_list) - - -def get_all_groups(h5_files): - """returns a dictionary of group_name: h5_path pairs from a list of h5_files.""" - if isinstance(h5_files, bytes) or isinstance(h5_files, str): - h5_files = [h5_files] - groups = {} - for path in h5_files: - for group_name in get_grouplist(path): - if group_name in groups: - raise ValueError('Error: group %s is defined in both %s and %s. ' % (group_name, groups[group_name], path) + - 'Only uniquely named groups can be used together ' - 'to make a run file.') - groups[group_name] = path - return groups - - -def get_globals(groups): - """Takes a dictionary of group_name: h5_file pairs and pulls the - globals out of the groups in their files. The globals are strings - storing python expressions at this point. All these globals are - packed into a new dictionary, keyed by group_name, where the values - are dictionaries which look like {global_name: (expression, units, expansion), ...}""" - # get a list of filepaths: - filepaths = set(groups.values()) - sequence_globals = {} - for filepath in filepaths: - groups_from_this_file = [g for g, f in groups.items() if f == filepath] - with h5py.File(filepath, 'r') as f: - for group_name in groups_from_this_file: - sequence_globals[group_name] = {} - globals_group = f['globals'][group_name] - values = dict(globals_group.attrs) - units = dict(globals_group['units'].attrs) - expansions = dict(globals_group['expansion'].attrs) - for global_name, value in values.items(): - unit = units[global_name] - expansion = expansions[global_name] - # Replace numpy strings with python unicode strings. - # DEPRECATED, for backward compat with old files - value = _ensure_str(value) - unit = _ensure_str(unit) - expansion = _ensure_str(expansion) - sequence_globals[group_name][global_name] = value, unit, expansion - return sequence_globals - - -def evaluate_globals(sequence_globals, raise_exceptions=True): - """Takes a dictionary of globals as returned by get_globals. These - globals are unevaluated strings. Evaluates them all in the same - namespace so that the expressions can refer to each other. Iterates - to allow for NameErrors to be resolved by subsequently defined - globals. Throws an exception if this does not result in all errors - going away. The exception contains the messages of all exceptions - which failed to be resolved. If raise_exceptions is False, any - evaluations resulting in an exception will instead return the - exception object in the results dictionary""" - - # Flatten all the groups into one dictionary of {global_name: - # expression} pairs. Also create the group structure of the results - # dict, which has the same structure as sequence_globals: - all_globals = {} - results = {} - expansions = {} - global_hierarchy = {} - # Pre-fill the results dictionary with groups, this is needed for - # storing exceptions in the case of globals with the same name being - # defined in multiple groups (all of them get the exception): - for group_name in sequence_globals: - results[group_name] = {} - multiply_defined_globals = set() - for group_name in sequence_globals: - for global_name in sequence_globals[group_name]: - if global_name in all_globals: - # The same global is defined twice. Either raise an - # exception, or store the exception for each place it is - # defined, depending on whether raise_exceptions is True: - groups_with_same_global = [] - for other_group_name in sequence_globals: - if global_name in sequence_globals[other_group_name]: - groups_with_same_global.append(other_group_name) - exception = ValueError('Global named \'%s\' is defined in multiple active groups:\n ' % global_name + - '\n '.join(groups_with_same_global)) - if raise_exceptions: - raise exception - for other_group_name in groups_with_same_global: - results[other_group_name][global_name] = exception - multiply_defined_globals.add(global_name) - all_globals[global_name], units, expansion = sequence_globals[group_name][global_name] - expansions[global_name] = expansion - - # Do not attempt to evaluate globals which are multiply defined: - for global_name in multiply_defined_globals: - del all_globals[global_name] - - # Eval the expressions in the same namespace as each other: - evaled_globals = {} - # we use a "TraceDictionary" to track which globals another global depends on - sandbox = TraceDictionary() - exec('from pylab import *', sandbox, sandbox) - exec('from runmanager.functions import *', sandbox, sandbox) - globals_to_eval = all_globals.copy() - previous_errors = -1 - while globals_to_eval: - errors = [] - for global_name, expression in globals_to_eval.copy().items(): - # start the trace to determine which globals this global depends on - sandbox.start_trace() - try: - code = compile(expression, '', 'eval') - value = eval(code, sandbox) - # Need to know the length of any generators, convert to tuple: - if isinstance(value, types.GeneratorType): - value = iterator_to_tuple(value) - # Make sure if we're zipping or outer-producting this value, that it can - # be iterated over: - if expansions[global_name] == 'outer': - try: - iter(value) - except Exception as e: - raise ExpansionError(str(e)) - except Exception as e: - # Don't raise, just append the error to a list, we'll display them all later. - errors.append((global_name, e)) - sandbox.stop_trace() - continue - # Put the global into the namespace so other globals can use it: - sandbox[global_name] = value - del globals_to_eval[global_name] - evaled_globals[global_name] = value - - # get the results from the global trace - trace_data = sandbox.stop_trace() - # Only store names of globals (not other functions) - for key in list(trace_data): # copy the list before iterating over it - if key not in all_globals: - trace_data.remove(key) - if trace_data: - global_hierarchy[global_name] = trace_data - - if len(errors) == previous_errors: - # Since some globals may refer to others, we expect maybe - # some NameErrors to have occured. There should be fewer - # NameErrors each iteration of this while loop, as globals - # that are required become defined. If there are not fewer - # errors, then there is something else wrong and we should - # raise it. - if raise_exceptions: - message = 'Error parsing globals:\n' - for global_name, exception in errors: - message += '%s: %s: %s\n' % (global_name, exception.__class__.__name__, str(exception)) - raise Exception(message) - else: - for global_name, exception in errors: - evaled_globals[global_name] = exception - break - previous_errors = len(errors) - - # Assemble results into a dictionary of the same format as sequence_globals: - for group_name in sequence_globals: - for global_name in sequence_globals[group_name]: - # Do not attempt to override exception objects already stored - # as the result of multiply defined globals: - if global_name not in results[group_name]: - results[group_name][global_name] = evaled_globals[global_name] - - return results, global_hierarchy, expansions - - -def expand_globals(sequence_globals, evaled_globals, expansion_config = None, return_dimensions = False): - """Expands iterable globals according to their expansion - settings. Creates a number of 'axes' which are to be outer product'ed - together. Some of these axes have only one element, these are globals - that do not vary. Some have a set of globals being zipped together, - iterating in lock-step. Others contain a single global varying - across its values (the globals set to 'outer' expansion). Returns - a list of shots, each element of which is a dictionary for that - shot's globals.""" - - if expansion_config is None: - order = {} - shuffle = {} - else: - order = {k:v['order'] for k,v in expansion_config.items() if 'order' in v} - shuffle = {k:v['shuffle'] for k,v in expansion_config.items() if 'shuffle' in v} - - values = {} - expansions = {} - for group_name in sequence_globals: - for global_name in sequence_globals[group_name]: - expression, units, expansion = sequence_globals[group_name][global_name] - value = evaled_globals[group_name][global_name] - values[global_name] = value - expansions[global_name] = expansion - - # Get a list of the zip keys in use: - zip_keys = set(expansions.values()) - try: - zip_keys.remove('outer') - except KeyError: - pass - - axes = {} - global_names = {} - dimensions = {} - for zip_key in zip_keys: - axis = [] - zip_global_names = [] - for global_name in expansions: - if expansions[global_name] == zip_key: - value = values[global_name] - if isinstance(value, Exception): - continue - if not zip_key: - # Wrap up non-iterating globals (with zip_key = '') in a - # one-element list. When zipped and then outer product'ed, - # this will give us the result we want: - value = [value] - axis.append(value) - zip_global_names.append(global_name) - axis = list(zip(*axis)) - dimensions['zip '+zip_key] = len(axis) - axes['zip '+zip_key] = axis - global_names['zip '+zip_key] = zip_global_names - - # Give each global being outer-product'ed its own axis. It gets - # wrapped up in a list and zipped with itself so that it is in the - # same format as the zipped globals, ready for outer-producting - # together: - for global_name in expansions: - if expansions[global_name] == 'outer': - value = values[global_name] - if isinstance(value, Exception): - continue - axis = [value] - axis = list(zip(*axis)) - dimensions['outer '+global_name] = len(axis) - axes['outer '+global_name] = axis - global_names['outer '+global_name] = [global_name] - - # add any missing items to order and dimensions - for key, value in axes.items(): - if key not in order: - order[key] = -1 - if key not in shuffle: - shuffle[key] = False - if key not in dimensions: - dimensions[key] = 1 - - # shuffle relevant axes - for axis_name, axis_values in axes.items(): - if shuffle[axis_name]: - random.shuffle(axis_values) - - # sort axes and global names by order - axes = [axes.get(key) for key in sorted(order, key=order.get)] - global_names = [global_names.get(key) for key in sorted(order, key=order.get)] - - # flatten the global names - global_names = [global_name for global_list in global_names for global_name in global_list] - - - shots = [] - for axis_values in itertools.product(*axes): - # values here is a tuple of tuples, with the outer list being over - # the axes. We need to flatten it to get our individual values out - # for each global, since we no longer care what axis they are on: - global_values = [value for axis in axis_values for value in axis] - shot_globals = dict(zip(global_names, global_values)) - shots.append(shot_globals) - - if return_dimensions: - return shots, dimensions - else: - return shots - -def next_sequence_index(shot_basedir, dt, increment=True): - """Return the next sequence index for sequences in the given base directory (i.e. - /) and the date of the given datetime - object, and increment the sequence index atomically on disk if increment=True. If - not setting increment=True, then the result is indicative only and may be used by - other code at any time. One must increment the sequence index prior to use.""" - from labscript_utils.ls_zprocess import Lock - from labscript_utils.shared_drive import path_to_agnostic - - DATE_FORMAT = '%Y-%m-%d' - # The file where we store the next sequence index on disk: - sequence_index_file = os.path.join(shot_basedir, '.next_sequence_index') - # Open with zlock to prevent race conditions with other code: - with Lock(path_to_agnostic(sequence_index_file), read_only=not increment): - try: - with open(sequence_index_file) as f: - datestr, sequence_index = json.load(f) - if datestr != dt.strftime(DATE_FORMAT): - # New day, start from zero again: - sequence_index = 0 - except (OSError, IOError) as exc: - if exc.errno != errno.ENOENT: - raise - # File doesn't exist yet, start from zero - sequence_index = 0 - if increment: - # Write the new file with the incremented sequence index - os.makedirs(os.path.dirname(sequence_index_file), exist_ok=True) - with open(sequence_index_file, 'w') as f: - json.dump([dt.strftime(DATE_FORMAT), sequence_index + 1], f) - return sequence_index - - -def new_sequence_details(script_path, config=None, increment_sequence_index=True): - """Generate the details for a new sequence: the toplevel attrs sequence_date, - sequence_index, sequence_id; and the the output directory and filename prefix for - the shot files, according to labconfig settings. If increment_sequence_index=True, - then we are claiming the resulting sequence index for use such that it cannot be - used by anyone else. This should be done if the sequence details are immediately - about to be used to compile a sequence. Otherwise, set increment_sequence_index to - False, but in that case the results are indicative only and one should call this - function again with increment_sequence_index=True before compiling the sequence, as - otherwise the sequence_index may be used by other code in the meantime.""" - if config is None: - config = LabConfig() - script_basename = os.path.splitext(os.path.basename(script_path))[0] - shot_storage = config.get('DEFAULT', 'experiment_shot_storage') - shot_basedir = os.path.join(shot_storage, script_basename) - now = datetime.datetime.now() - sequence_timestamp = now.strftime('%Y%m%dT%H%M%S') - - # Toplevel attributes to be saved to the shot files: - sequence_date = now.strftime('%Y-%m-%d') - sequence_id = sequence_timestamp + '_' + script_basename - sequence_index = next_sequence_index(shot_basedir, now, increment_sequence_index) - - sequence_attrs = { - 'script_basename': script_basename, - 'sequence_date': sequence_date, - 'sequence_index': sequence_index, - 'sequence_id': sequence_id, - } - - # Compute the output directory based on labconfig settings: - try: - subdir_format = config.get('runmanager', 'output_folder_format') - except (LabConfig.NoOptionError, LabConfig.NoSectionError): - subdir_format = os.path.join('%Y', '%m', '%d', '{sequence_index:05d}') - - # Format the output directory according to the current timestamp, sequence index and - # sequence_timestamp, if present in the format string: - subdir = now.strftime(subdir_format).format( - sequence_index=sequence_index, sequence_timestamp=sequence_timestamp - ) - shot_output_dir = os.path.join(shot_basedir, subdir) - - # Compute the shot filename prefix according to labconfig settings: - try: - filename_prefix_format = config.get('runmanager', 'filename_prefix_format') - except (LabConfig.NoOptionError, LabConfig.NoSectionError): - # Default, for backward compatibility: - filename_prefix_format = '{sequence_timestamp}_{script_basename}' - # Format the filename prefix according to the current timestamp, sequence index, - # sequence_timestamp, and script_basename, if present in the format string: - filename_prefix = now.strftime(filename_prefix_format).format( - sequence_index=sequence_index, - sequence_timestamp=sequence_timestamp, - script_basename=script_basename, - ) - - return sequence_attrs, shot_output_dir, filename_prefix - - -def make_run_files( - output_folder, - sequence_globals, - shots, - sequence_attrs, - filename_prefix, - shuffle=False, -): - """Does what it says. sequence_globals and shots are of the datatypes returned by - get_globals and get_shots, one is a nested dictionary with string values, and the - other a flat dictionary. sequence_attrs is a dict of the attributes pertaining to - this sequence to be initially set at the top-level group of the h5 file, as returned - by new_sequence_details. output_folder and filename_prefix determine the directory - shot files will be output to, as well as their filenames (this function will - generate filenames with the shot number and .h5 extension appended to - filename_prefix). Sensible defaults for these are also returned by - new_sequence_details(), so preferably these should be used. - - Shuffle will randomise the order that the run files are generated in with respect to - which element of shots they come from. This function returns a *generator*. The run - files are not actually created until you loop over this generator (which gives you - the filepaths). This is useful for not having to clean up as many unused files in - the event of failed compilation of labscripts. If you want all the run files to be - created at some point, simply convert the returned generator to a list. The - filenames the run files are given is simply the sequence_id with increasing integers - appended.""" - basename = os.path.join(output_folder, filename_prefix) - nruns = len(shots) - ndigits = int(np.ceil(np.log10(nruns))) - if shuffle: - random.shuffle(shots) - for i, shot_globals in enumerate(shots): - runfilename = ('%s_%0' + str(ndigits) + 'd.h5') % (basename, i) - make_single_run_file( - runfilename, sequence_globals, shot_globals, sequence_attrs, i, nruns - ) - yield runfilename - - -def make_single_run_file(filename, sequenceglobals, runglobals, sequence_attrs, run_no, n_runs): - """Does what it says. runglobals is a dict of this run's globals, the format being - the same as that of one element of the list returned by expand_globals. - sequence_globals is a nested dictionary of the type returned by get_globals. - sequence_attrs is a dict of attributes pertaining to this sequence, as returned by - new_sequence_details. run_no and n_runs must be provided, if this run file is part - of a sequence, then they should reflect how many run files are being generated in - this sequence, all of which must have identical sequence_attrs.""" - os.makedirs(os.path.dirname(filename), exist_ok=True) - with h5py.File(filename, 'w') as f: - f.attrs.update(sequence_attrs) - f.attrs['run number'] = run_no - f.attrs['n_runs'] = n_runs - f.create_group('globals') - if sequenceglobals is not None: - for groupname, groupvars in sequenceglobals.items(): - group = f['globals'].create_group(groupname) - unitsgroup = group.create_group('units') - expansiongroup = group.create_group('expansion') - for name, (value, units, expansion) in groupvars.items(): - group.attrs[name] = value - unitsgroup.attrs[name] = units - expansiongroup.attrs[name] = expansion - for name, value in runglobals.items(): - if value is None: - # Store it as a null object reference: - value = h5py.Reference() - try: - f['globals'].attrs[name] = value - except Exception as e: - message = ('Global %s cannot be saved as an hdf5 attribute. ' % name + - 'Globals can only have relatively simple datatypes, with no nested structures. ' + - 'Original error was:\n' + - '%s: %s' % (e.__class__.__name__, str(e))) - raise ValueError(message) - - -def make_run_file_from_globals_files(labscript_file, globals_files, output_path, config=None): - """Creates a run file output_path, using all the globals from globals_files. Uses - labscript_file to determine the sequence_attrs only""" - groups = get_all_groups(globals_files) - sequence_globals = get_globals(groups) - evaled_globals, global_hierarchy, expansions = evaluate_globals(sequence_globals) - shots = expand_globals(sequence_globals, evaled_globals) - if len(shots) > 1: - scanning_globals = [] - for global_name in expansions: - if expansions[global_name]: - scanning_globals.append(global_name) - raise ValueError('Cannot compile to a single run file: The following globals are a sequence: ' + - ', '.join(scanning_globals)) - - sequence_attrs, _, _ = new_sequence_details( - labscript_file, config=config, increment_sequence_index=True - ) - make_single_run_file(output_path, sequence_globals, shots[0], sequence_attrs, 1, 1) - - -def compile_labscript_async(labscript_file, run_file, - stream_port=None, done_callback=None): - """Compiles labscript_file with run_file. - - This function is designed to be called in a thread. - The stdout and stderr from the compilation will be shovelled into - stream_port via zmq push as it spews forth, and when compilation is complete, - done_callback will be called with a boolean argument indicating success. Note that - the zmq communication will be encrypted, or not, according to security settings in - labconfig. If you want to receive the data on a zmq socket, do so using a PULL - socket created from a :class:`labscript_utils.ls_zprocess.Context`, or using a - :class:`labscript_utils.ls_zprocess.ZMQServer`. These subclasses will also be configured - with the appropriate security settings and will be able to receive the messages. - - Args: - labscript_file (str): Path to labscript file to be compiled - run_file (str): Path to h5 file where compilation output is stored. - This file must already exist with proper globals initialization. - See :func:`new_globals_file` for details. - stream_port (zmq.socket, optional): ZMQ socket to push stdout and stderr. - If None, defaults to calling process stdout/stderr. Default is None. - done_callback (function, optional): Callback function run when compilation finishes. - Takes a single boolean argument marking compilation success or failure. - If None, callback is skipped. Default is None. - """ - compiler_path = os.path.join(os.path.dirname(__file__), 'batch_compiler.py') - to_child, from_child, child = process_tree.subprocess( - compiler_path, output_redirection_port=stream_port - ) - to_child.put(['compile', [labscript_file, run_file]]) - while True: - signal, data = from_child.get() - if signal == 'done': - success = data - to_child.put(['quit', None]) - child.communicate() - if done_callback is not None: - done_callback(success) - break - else: - raise RuntimeError((signal, data)) - - -def compile_multishot_async(labscript_file, run_files, - stream_port=None, done_callback=None): - """Compiles labscript_file with multiple run_files (ie globals). - - This function is designed to be called in a thread. - The stdout and stderr from the compilation will be shovelled into - stream_port via zmq push as it spews forth, and when each compilation is complete, - done_callback will be called with a boolean argument indicating success. Compilation - will stop after the first failure. If you want to receive the data on a zmq socket, - do so using a PULL socket created from a :class:`labscript_utils.ls_zprocess.Context`, or - using a :class:`labscript_utils.ls_zprocess.ZMQServer`. These subclasses will also be - configured with the appropriate security settings and will be able to receive the - messages. - - Args: - labscript_file (str): Path to labscript file to be compiled - run_files (list of str): Paths to h5 file where compilation output is stored. - These files must already exist with proper globals initialization. - stream_port (zmq.socket, optional): ZMQ socket to push stdout and stderr. - If None, defaults to calling process stdout/stderr. Default is None. - done_callback (function, optional): Callback function run when compilation finishes. - Takes a single boolean argument marking compilation success or failure. - If None, callback is skipped. Default is None. - """ - compiler_path = os.path.join(os.path.dirname(__file__), 'batch_compiler.py') - to_child, from_child, child = process_tree.subprocess( - compiler_path, output_redirection_port=stream_port - ) - try: - for run_file in run_files: - to_child.put(['compile', [labscript_file, run_file]]) - while True: - signal, data = from_child.get() - if signal == 'done': - success = data - if done_callback is not None: - done_callback(data) - break - if not success: - break - except Exception: - error = traceback.format_exc() - zmq_push_multipart(stream_port, data=[b'stderr', error.encode('utf-8')]) - to_child.put(['quit', None]) - child.communicate() - raise - to_child.put(['quit', None]) - child.communicate() - - -def compile_labscript_with_globals_files_async(labscript_file, globals_files, output_path, - stream_port, done_callback): - """Compiles labscript_file with multiple globals files into a directory. - - Instead, stderr and stdout will be put to - stream_port via zmq push in the multipart message format `['stdout','hello, world\\n']` - etc. When compilation is finished, the function done_callback will be called a - boolean argument indicating success or failure. If you want to receive the data on - a zmq socket, do so using a PULL socket created from a - :external:class:`labscript_utils.ls_zprocess.Context`, or using a - :external:class:`labscript_utils.ls_zprocess.ZMQServer`. These subclasses will also be configured with - the appropriate security settings and will be able to receive the messages. - - Args: - labscript_file (str): Path to labscript file to be compiled - globals_files (list of str): Paths to h5 file where globals values to be used are stored. - See :func:`make_run_file_from_globals_files` for details. - output_path (str): Folder to save compiled h5 files to. - stream_port (zmq.socket): ZMQ socket to push stdout and stderr. - done_callback (function): Callback function run when compilation finishes. - Takes a single boolean argument marking compilation success or failure. - """ - try: - make_run_file_from_globals_files(labscript_file, globals_files, output_path) - thread = threading.Thread( - target=compile_labscript_async, args=[labscript_file, output_path, stream_port, done_callback]) - thread.daemon = True - thread.start() - except Exception: - error = traceback.format_exc() - zmq_push_multipart(stream_port, data=[b'stderr', error.encode('utf-8')]) - t = threading.Thread(target=done_callback, args=(False,)) - t.daemon = True - t.start() - - -def get_shot_globals(filepath): - """Returns the evaluated globals for a shot, for use by labscript or lyse. - Simple dictionary access as in dict(h5py.File(filepath).attrs) would be fine - except we want to apply some hacks, so it's best to do that in one place. - - Deprecated: use identical function `labscript_utils.shot_utils.get_shot_globals` - """ - - warnings.warn( - FutureWarning("get_shot_globals has moved to labscript_utils.shot_utils. " - "Please update your code to import it from there.")) - - return labscript_utils.shot_utils.get_shot_globals(filepath) - - -def dict_diff(dict1, dict2): - """Return the difference between two dictionaries as a dictionary of key: [val1, val2] pairs. - Keys unique to either dictionary are included as key: [val1, '-'] or key: ['-', val2].""" - diff_keys = [] - common_keys = np.intersect1d(list(dict1.keys()), list(dict2.keys())) - for key in common_keys: - if np.iterable(dict1[key]) or np.iterable(dict2[key]): - if not np.array_equal(dict1[key], dict2[key]): - diff_keys.append(key) - else: - if dict1[key] != dict2[key]: - diff_keys.append(key) - - dict1_unique = [key for key in dict1.keys() if key not in common_keys] - dict2_unique = [key for key in dict2.keys() if key not in common_keys] - - diff = {} - for key in diff_keys: - diff[key] = [dict1[key], dict2[key]] - - for key in dict1_unique: - diff[key] = [dict1[key], '-'] - - for key in dict2_unique: - diff[key] = ['-', dict2[key]] - - return diff - - -def find_comments(src): - """Return a list of start and end indices for where comments are in given Python - source. Comments on separate lines with only whitespace in between them are - coalesced. Whitespace preceding a comment is counted as part of the comment.""" - line_start = 0 - comments = [] - tokens = tokenize.generate_tokens(io.StringIO(src).readline) - try: - for token_type, token_value, (_, start), (_, end), _ in tokens: - if token_type == tokenize.COMMENT: - comments.append((line_start + start, line_start + end)) - if token_value == '\n': - line_start += end - except tokenize.TokenError: - pass - # coalesce comments with only whitespace between them: - to_merge = [] - for i, ((start1, end1), (start2, end2)) in enumerate(zip(comments, comments[1:])): - if not src[end1:start2].strip(): - to_merge.append(i) - # Reverse order so deletion doesn't change indices: - for i in reversed(to_merge): - start1, end1 = comments[i] - start2, end2 = comments[i + 1] - comments[i] = (start1, end2) - del comments[i + 1] - # Extend each comment block to the left to include whitespace: - for i, (start, end) in enumerate(comments): - n_whitespace_chars = len(src[:start]) - len(src[:start].rstrip()) - comments[i] = start - n_whitespace_chars, end - # Extend the final comment to the right to include whitespace: - if comments: - start, end = comments[-1] - n_whitespace_chars = len(src[end:]) - len(src[end:].rstrip()) - comments[-1] = (start, end + n_whitespace_chars) - return comments - - -def remove_comments_and_tokenify(src): - """Removes comments from source code, leaving it otherwise intact, - and returns it. Also returns the raw tokens for the code, allowing - comparisons between source to be made without being sensitive to - whitespace.""" - # Remove comments - for (start, end) in reversed(find_comments(src)): - src = src[:start] + src[end:] - # Tokenify: - tokens = [] - tokens_iter = tokenize.generate_tokens(io.StringIO(src).readline) - try: - for _, token_value, _, _, _ in tokens_iter: - if token_value: - tokens.append(token_value) - except tokenize.TokenError: - pass - return src, tokens - - -def flatten_globals(sequence_globals, evaluated=False): - """Flattens the data structure of the globals. If evaluated=False, - saves only the value expression string of the global, not the - units or expansion.""" - flattened_sequence_globals = {} - for globals_group in sequence_globals.values(): - for name, value in globals_group.items(): - if evaluated: - flattened_sequence_globals[name] = value - else: - value_expression, units, expansion = value - flattened_sequence_globals[name] = value_expression - return flattened_sequence_globals - - -def globals_diff_groups(active_groups, other_groups, max_cols=1000, return_string=True): - """Given two sets of globals groups, perform a diff of the raw - and evaluated globals.""" - our_sequence_globals = get_globals(active_groups) - other_sequence_globals = get_globals(other_groups) - - # evaluate globals - our_evaluated_sequence_globals, _, _ = evaluate_globals(our_sequence_globals, raise_exceptions=False) - other_evaluated_sequence_globals, _, _ = evaluate_globals(other_sequence_globals, raise_exceptions=False) - - # flatten globals dictionaries - our_globals = flatten_globals(our_sequence_globals, evaluated=False) - other_globals = flatten_globals(other_sequence_globals, evaluated=False) - our_evaluated_globals = flatten_globals(our_evaluated_sequence_globals, evaluated=True) - other_evaluated_globals = flatten_globals(other_evaluated_sequence_globals, evaluated=True) - - # diff the *evaluated* globals - value_differences = dict_diff(other_evaluated_globals, our_evaluated_globals) - - # We are interested only in displaying globals where *both* the - # evaluated global *and* its unevaluated expression (ignoring comments - # and whitespace) differ. This will minimise false positives where a - # slight change in an expression still leads to the same value, or - # where an object has a poorly defined equality operator that returns - # False even when the two objects are identical. - filtered_differences = {} - for name, (other_value, our_value) in value_differences.items(): - our_expression = our_globals.get(name, '-') - other_expression = other_globals.get(name, '-') - # Strip comments, get tokens so we can diff without being sensitive to comments or whitespace: - our_expression, our_tokens = remove_comments_and_tokenify(our_expression) - other_expression, other_tokens = remove_comments_and_tokenify(other_expression) - if our_tokens != other_tokens: - filtered_differences[name] = [repr(other_value), repr(our_value), other_expression, our_expression] - if filtered_differences: - import pandas as pd - df = pd.DataFrame.from_dict(filtered_differences, 'index') - df = df.sort_index() - df.columns = ['Prev (Eval)', 'Current (Eval)', 'Prev (Raw)', 'Current (Raw)'] - df_string = df.to_string(max_cols=max_cols) - payload = df_string + '\n\n' - else: - payload = 'Evaluated globals are identical to those of selected file.\n' - if return_string: - return payload - else: - print(payload) - return df - - -def globals_diff_shots(file1, file2, max_cols=100): - # Get file's globals groups - active_groups = get_all_groups(file1) - - # Get other file's globals groups - other_groups = get_all_groups(file2) - - print('Globals diff between:\n%s\n%s\n\n' % (file1, file2)) - return globals_diff_groups(active_groups, other_groups, max_cols=max_cols, return_string=False) diff --git a/runmanager/__main__.py b/runmanager/__main__.py index dc9e255..51fc4f1 100644 --- a/runmanager/__main__.py +++ b/runmanager/__main__.py @@ -61,6 +61,12 @@ from zprocess import raise_exception_in_thread import runmanager import runmanager.remote +import runmanager.compiler as compiler +import runmanager.differ as differ +import runmanager.tokenizer as tokenizer +import runmanager.group_manager as group_manager +from runmanager.exceptions import ExpansionError +from runmanager.widgets import RunmanagerColors, FingerTabWidget, TableView, TreeView, TabToolButton, AlternatingColorModel from qtutils import ( inmain, @@ -102,18 +108,6 @@ def log_if_global(g, g_list, message): logger.info(message) # uses global `logger` instance defined in __main__ script section -def composite_colors(r0, g0, b0, a0, r1, g1, b1, a1): - """composite a second colour over a first with given alpha values and return the - result""" - a0 /= 255 - a1 /= 255 - a = a0 + a1 - a0 * a1 - r = (a1 * r1 + (1 - a1) * a0 * r0) / a - g = (a1 * g1 + (1 - a1) * a0 * g0) / a - b = (a1 * b1 + (1 - a1) * a0 * b0) / a - return [int(round(x)) for x in (r, g, b, 255 * a)] - - @inmain_decorator() def error_dialog(message): QtWidgets.QMessageBox.warning(app.ui, 'runmanager', message) @@ -150,607 +144,6 @@ def scroll_view_to_row_if_current(view, item): horizontal_scrollbar.setValue(existing_horizontal_position) -class FingerTabBarWidget(QtWidgets.QTabBar): - - """A TabBar with the tabs on the left and the text horizontal. Credit to - @LegoStormtroopr, https://gist.github.com/LegoStormtroopr/5075267. We will - promote the TabBar from the ui file to one of these.""" - - def __init__(self, parent=None, minwidth=180, minheight=30, **kwargs): - QtWidgets.QTabBar.__init__(self, parent, **kwargs) - self.minwidth = minwidth - self.minheight = minheight - self.iconPosition = kwargs.pop('iconPosition', QtWidgets.QTabWidget.West) - self._movable = None - self.tab_movable = {} - self.paint_clip = None - - def setMovable(self, movable, index=None): - """Set tabs movable on an individual basis, or set for all tabs if no - index specified""" - if index is None: - self._movable = movable - self.tab_movable = {} - QtWidgets.QTabBar.setMovable(self, movable) - else: - self.tab_movable[int(index)] = bool(movable) - - def isMovable(self, index=None): - if index is None: - if self._movable is None: - self._movable = QtWidgets.QTabBar.isMovable(self) - return self._movable - return self.tab_movable.get(index, self._movable) - - def indexAtPos(self, point): - for index in range(self.count()): - if self.tabRect(index).contains(point): - return index - - def mouseEventIndex(self, event): - if QT_ENV == 'PyQt5': - return self.indexAtPos(event.pos()) - else: - # Qt6 position returns QPointF instead of QPoint - return self.indexAtPos(event.position().toPoint()) - - def mousePressEvent(self, event): - index = self.mouseEventIndex(event) - if not self.tab_movable.get(index, self.isMovable()): - QtWidgets.QTabBar.setMovable(self, False) # disable dragging until they release the mouse - return QtWidgets.QTabBar.mousePressEvent(self, event) - - def mouseReleaseEvent(self, event): - if self.isMovable(): - # Restore this in case it was temporarily disabled by mousePressEvent - QtWidgets.QTabBar.setMovable(self, True) - return QtWidgets.QTabBar.mouseReleaseEvent(self, event) - - def tabLayoutChange(self): - total_height = 0 - for index in range(self.count()): - tabRect = self.tabRect(index) - total_height += tabRect.height() - if total_height > self.parent().height(): - # Don't paint over the top of the scroll buttons: - scroll_buttons_area_height = 2*max(self.style().pixelMetric(QtWidgets.QStyle.PM_TabBarScrollButtonWidth), - self.style().pixelMetric(QtWidgets.QStyle.PM_LayoutHorizontalSpacing)) - self.paint_clip = self.width(), self.parent().height() - scroll_buttons_area_height - else: - self.paint_clip = None - - def paintEvent(self, event): - painter = QtWidgets.QStylePainter(self) - if self.paint_clip is not None: - painter.setClipRect(0, 0, *self.paint_clip) - - option = QtWidgets.QStyleOptionTab() - for index in range(self.count()): - tabRect = self.tabRect(index) - self.initStyleOption(option, index) - painter.drawControl(QtWidgets.QStyle.CE_TabBarTabShape, option) - if not self.tabIcon(index).isNull(): - icon = self.tabIcon(index).pixmap(self.iconSize()) - alignment = QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter - tabRect.moveLeft(10) - painter.drawItemPixmap(tabRect, alignment, icon) - tabRect.moveLeft(self.iconSize().width() + 15) - else: - tabRect.moveLeft(10) - painter.drawText(tabRect, QtCore.Qt.AlignVCenter, self.tabText(index)) - if self.paint_clip is not None: - x_clip, y_clip = self.paint_clip - painter.setClipping(False) - palette = self.palette() - mid_color = palette.color(QtGui.QPalette.Mid) - painter.setPen(mid_color) - painter.drawLine(0, y_clip, x_clip, y_clip) - painter.end() - - - def tabSizeHint(self, index): - fontmetrics = QtGui.QFontMetrics(self.font()) - text_size = fontmetrics.size(QtCore.Qt.TextSingleLine, self.tabText(index)) - text_width = text_size.width() - text_height = text_size.height() - height = text_height + 15 - height = max(self.minheight, height) - width = text_width + 15 - - button = self.tabButton(index, QtWidgets.QTabBar.RightSide) - if button is not None: - height = max(height, button.height() + 7) - # Same amount of space around the button horizontally as it has vertically: - width += button.width() + height - button.height() - width = max(self.minwidth, width) - return QtCore.QSize(width, height) - - def setTabButton(self, index, geometry, button): - if not isinstance(button, TabToolButton): - raise TypeError('Not a TabToolButton, won\'t paint correctly. Use a TabToolButton') - result = QtWidgets.QTabBar.setTabButton(self, index, geometry, button) - button.move(*button.get_correct_position()) - return result - - -class TabToolButton(QtWidgets.QToolButton): - def __init__(self, *args, **kwargs): - QtWidgets.QToolButton.__init__(self, *args, **kwargs) - self.setFocusPolicy(QtCore.Qt.NoFocus) - - def paintEvent(self, event): - painter = QtWidgets.QStylePainter(self) - paint_clip = self.parent().paint_clip - if paint_clip is not None: - point = QtCore.QPoint(*paint_clip) - global_point = self.parent().mapToGlobal(point) - local_point = self.mapFromGlobal(global_point) - painter.setClipRect(0, 0, local_point.x(), local_point.y()) - option = QtWidgets.QStyleOptionToolButton() - self.initStyleOption(option) - painter.drawComplexControl(QtWidgets.QStyle.CC_ToolButton, option) - - def get_correct_position(self): - parent = self.parent() - for index in range(parent.count()): - if parent.tabButton(index, QtWidgets.QTabBar.RightSide) is self: - break - else: - raise LookupError('Tab not found') - tabRect = parent.tabRect(index) - tab_x, tab_y, tab_width, tab_height = tabRect.x(), tabRect.y(), tabRect.width(), tabRect.height() - size = self.sizeHint() - width = size.width() - height = size.height() - padding = int((tab_height - height) / 2) - correct_x = tab_x + tab_width - width - padding - correct_y = tab_y + padding - return correct_x, correct_y - - def moveEvent(self, event): - try: - correct_x, correct_y = self.get_correct_position() - except LookupError: - return # Things aren't initialised yet - if self.x() != correct_x or self.y() != correct_y: - # Move back! I shall not be moved! - self.move(correct_x, correct_y) - return QtWidgets.QToolButton.moveEvent(self, event) - - -class FingerTabWidget(QtWidgets.QTabWidget): - - """A QTabWidget equivalent which uses our FingerTabBarWidget""" - - def __init__(self, parent, *args): - QtWidgets.QTabWidget.__init__(self, parent, *args) - self.setTabBar(FingerTabBarWidget(self)) - - def addTab(self, *args, **kwargs): - closeable = kwargs.pop('closable', False) - index = QtWidgets.QTabWidget.addTab(self, *args, **kwargs) - self.setTabClosable(index, closeable) - return index - - def setTabClosable(self, index, closable): - right_button = self.tabBar().tabButton(index, QtWidgets.QTabBar.RightSide) - if closable: - if not right_button: - # Make one: - close_button = TabToolButton(self.parent()) - close_button.setIcon(QtGui.QIcon(':/qtutils/fugue/cross')) - self.tabBar().setTabButton(index, QtWidgets.QTabBar.RightSide, close_button) - close_button.clicked.connect(lambda: self._on_close_button_clicked(close_button)) - else: - if right_button: - # Get rid of it: - self.tabBar().setTabButton(index, QtWidgets.QTabBar.RightSide, None) - - def _on_close_button_clicked(self, button): - for index in range(self.tabBar().count()): - if self.tabBar().tabButton(index, QtWidgets.QTabBar.RightSide) is button: - self.tabCloseRequested.emit(index) - break - - -class ItemView(object): - """Mixin for QTableView and QTreeView that emits a custom signal leftClicked(index) - after a left click on a valid index, and doubleLeftClicked(index) (in addition) on - double click. Also has modified tab and arrow key behaviour and custom selection - highlighting.""" - leftClicked = Signal(QtCore.QModelIndex) - doubleLeftClicked = Signal(QtCore.QModelIndex) - - - def __init__(self, *args): - super(ItemView, self).__init__(*args) - self._pressed_index = None - self._double_click = False - self.setAutoScroll(False) - palette = self.palette() - for group in [QtGui.QPalette.Active, QtGui.QPalette.Inactive]: - palette.setColor( - group, - QtGui.QPalette.Highlight, - QtGui.QColor(RunmanagerColors().COLOR_HIGHLIGHT) - ) - palette.setColor( - group, - QtGui.QPalette.HighlightedText, - palette.color(QtGui.QPalette.WindowText) - ) - self.setPalette(palette) - - def mouseEventIndex(self, event): - if QT_ENV == 'PyQt5': - return self.indexAt(event.pos()) - else: - # Qt6 returns QPointF instead of QPoint - return self.indexAt(event.position().toPoint()) - - def mousePressEvent(self, event): - result = super(ItemView, self).mousePressEvent(event) - index = self.mouseEventIndex(event) - if event.button() == QtCore.Qt.LeftButton and index.isValid(): - self._pressed_index = self.mouseEventIndex(event) - return result - - def leaveEvent(self, event): - result = super(ItemView, self).leaveEvent(event) - self._pressed_index = None - self._double_click = False - return result - - def mouseDoubleClickEvent(self, event): - # Ensure our left click event occurs regardless of whether it is the - # second click in a double click or not - result = super(ItemView, self).mouseDoubleClickEvent(event) - index = self.mouseEventIndex(event) - if event.button() == QtCore.Qt.LeftButton and index.isValid(): - self._pressed_index = self.mouseEventIndex(event) - self._double_click = True - return result - - def mouseReleaseEvent(self, event): - result = super(ItemView, self).mouseReleaseEvent(event) - index = self.mouseEventIndex(event) - if event.button() == QtCore.Qt.LeftButton and index.isValid() and index == self._pressed_index: - self.leftClicked.emit(index) - if self._double_click: - self.doubleLeftClicked.emit(index) - self._pressed_index = None - self._double_click = False - return result - - def keyPressEvent(self, event): - if event.key() in [QtCore.Qt.Key_Space, QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return]: - item = self.model().itemFromIndex(self.currentIndex()) - if item.isEditable(): - # Space/enter edits editable items: - self.edit(self.currentIndex()) - else: - # Space/enter on non-editable items simulates a left click: - self.leftClicked.emit(self.currentIndex()) - return super(ItemView, self).keyPressEvent(event) - - def moveCursor(self, cursor_action, keyboard_modifiers): - current_index = self.currentIndex() - current_row, current_column = current_index.row(), current_index.column() - if cursor_action == QtWidgets.QAbstractItemView.MoveUp: - return current_index.sibling(current_row - 1, current_column) - elif cursor_action == QtWidgets.QAbstractItemView.MoveDown: - return current_index.sibling(current_row + 1, current_column) - elif cursor_action == QtWidgets.QAbstractItemView.MoveLeft: - return current_index.sibling(current_row, current_column - 1) - elif cursor_action == QtWidgets.QAbstractItemView.MoveRight: - return current_index.sibling(current_row, current_column + 1) - elif cursor_action == QtWidgets.QAbstractItemView.MovePrevious: - return current_index.sibling(current_row, current_column - 1) - elif cursor_action == QtWidgets.QAbstractItemView.MoveNext: - return current_index.sibling(current_row, current_column + 1) - else: - return super(ItemView, self).moveCursor(cursor_action, keyboard_modifiers) - - -class TreeView(ItemView, QtWidgets.QTreeView): - """Treeview version of our customised ItemView""" - def __init__(self, parent=None): - super(TreeView, self).__init__(parent) - # Set columns to their minimum size, disabling resizing. Caller may still - # configure a specific section to stretch: - self.header().setSectionResizeMode( - QtWidgets.QHeaderView.ResizeToContents - ) - self.setItemDelegate(ItemDelegate(self)) - - -class TableView(ItemView, QtWidgets.QTableView): - """TableView version of our customised ItemView""" - def __init__(self, parent=None): - super(TableView, self).__init__(parent) - # Set rows and columns to the minimum size, disabling interactive resizing. - # Caller may still configure a specific column to stretch: - self.verticalHeader().setSectionResizeMode( - QtWidgets.QHeaderView.ResizeToContents - ) - self.horizontalHeader().setSectionResizeMode( - QtWidgets.QHeaderView.ResizeToContents - ) - self.horizontalHeader().sectionResized.connect(self.on_column_resized) - self.setItemDelegate(ItemDelegate(self)) - self.verticalHeader().hide() - self.setShowGrid(False) - self.horizontalHeader().setHighlightSections(False) - - def on_column_resized(self, col): - for row in range(self.model().rowCount()): - self.resizeRowToContents(row) - - -class AlternatingColorModel(QtGui.QStandardItemModel): - - def __init__(self, view): - QtGui.QStandardItemModel.__init__(self) - # How much darker in each channel is the alternate base color compared - # to the base color? - self.view = view - - # A cache, store brushes so we don't have to recalculate them. Is faster. - self.bg_brushes = {} - - def get_bgbrush(self, normal_brush, alternate, selected): - """Get cell colour as a function of its ordinary colour, whether it is on an odd - row, and whether it is selected.""" - normal_rgb = normal_brush.color().getRgb() if normal_brush is not None else None - try: - return self.bg_brushes[normal_rgb, alternate, selected] - except KeyError: - pass - # Get the colour of the cell with alternate row shading: - normal_color = self.view.palette().color( - QtGui.QPalette.ColorGroup.Disabled, - QtGui.QPalette.ColorRole.Base - ) - alternate_color = self.view.palette().color( - QtGui.QPalette.ColorGroup.Disabled, - QtGui.QPalette.ColorRole.AlternateBase - ) - if normal_rgb is None: - # No colour has been set. Use palette colours: - if alternate: - bg_color = alternate_color - else: - bg_color = normal_color - else: - bg_color = normal_brush.color() - if alternate: - # Modify alternate rows: - r, g, b, a = normal_rgb - nr, ng, nb, na = normal_color.getRgb() - ar, ag, ab, aa = alternate_color.getRgb() - alt_r = min(max(r + ar - nr, 0), 255) - alt_g = min(max(g + ag - ng, 0), 255) - alt_b = min(max(b + ab - nb, 0), 255) - alt_a = min(max(a + aa - na, 0), 255) - bg_color = QtGui.QColor(alt_r, alt_g, alt_b, alt_a) - - # If parent is a TableView, we handle selection highlighting as part of the - # background colours: - if selected and isinstance(self.view, QtWidgets.QTableView): - # Overlay highlight colour: - r_s, g_s, b_s, a_s = QtGui.QColor(RunmanagerColors().COLOR_HIGHLIGHT).getRgb() - r_0, g_0, b_0, a_0 = bg_color.getRgb() - rgb = composite_colors(r_0, g_0, b_0, a_0, r_s, g_s, b_s, a_s) - bg_color = QtGui.QColor(*rgb) - - brush = QtGui.QBrush(bg_color) - self.bg_brushes[normal_rgb, alternate, selected] = brush - return brush - - def data(self, index, role): - """When background color data is being requested, returns modified colours for - every second row, according to the palette of the view. This has the effect of - making the alternate colours visible even when custom colors have been set - the - same shading will be applied to the custom colours. Only really looks sensible - when the normal and alternate colors are similar. Also applies selection - highlight colour (using RunmanagerColors().COLOR_HIGHLIGHT), similarly with alternate-row - shading, for the case of a QTableView.""" - if role == QtCore.Qt.BackgroundRole: - normal_brush = QtGui.QStandardItemModel.data(self, index, QtCore.Qt.BackgroundRole) - selected = index in self.view.selectedIndexes() - alternate = index.row() % 2 - return self.get_bgbrush(normal_brush, alternate, selected) - return QtGui.QStandardItemModel.data(self, index, role) - - -class Editor(QtWidgets.QTextEdit): - """Popup editor with word wrapping and automatic resizing.""" - - def __init__(self, parent): - QtWidgets.QTextEdit.__init__(self, parent) - self.setWordWrapMode(QtGui.QTextOption.WordWrap) - self.setAcceptRichText(False) - self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff) - self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff) - self.textChanged.connect(self.update_size) - self.initial_height = None - palette = self.palette() - for group in [QtGui.QPalette.Active, QtGui.QPalette.Inactive]: - palette.setColor( - group, - QtGui.QPalette.Highlight, - QtGui.QColor(RunmanagerColors().COLOR_HIGHLIGHT) - ) - palette.setColor( - group, - QtGui.QPalette.HighlightedText, - palette.color(QtGui.QPalette.WindowText) - ) - self.setPalette(palette) - - def update_size(self): - if self.initial_height is not None: - # Temporarily shrink back to the initial height, just so that the document - # size below returns the preferred size rather than the current size. - # QTextDocument doesn't have a sizeHint of minimumSizeHint method, so this - # is the best we can do to get its minimum size. - self.setFixedHeight(self.initial_height) - preferred_height = self.document().size().toSize().height() - # Do not shrink smaller than the initial height: - if self.initial_height is not None and preferred_height >= self.initial_height: - self.setFixedHeight(preferred_height) - - def resizeEvent(self, event): - result = QtWidgets.QTextEdit.resizeEvent(self, event) - # Record the initial height after it is first set: - if self.initial_height is None: - self.initial_height = self.height() - return result - - - -class ItemDelegate(QtWidgets.QStyledItemDelegate): - - """An item delegate with a larger row height and column width, faint grey vertical - lines between columns, and a custom editor for handling multi-line data""" - MIN_ROW_HEIGHT = 22 - EXTRA_ROW_HEIGHT = 6 - EXTRA_COL_WIDTH = 20 - - def __init__(self, *args, **kwargs): - QtWidgets.QStyledItemDelegate.__init__(self, *args, **kwargs) - self._pen = QtGui.QPen() - self._pen.setWidth(1) - self._pen.setColor(QtGui.QColor.fromRgb(128, 128, 128, 64)) - - def sizeHint(self, *args): - size = QtWidgets.QStyledItemDelegate.sizeHint(self, *args) - if size.height() <= self.MIN_ROW_HEIGHT: - height = self.MIN_ROW_HEIGHT - else: - # Esnure cells with multiple lines of text still have some padding: - height = size.height() + self.EXTRA_ROW_HEIGHT - return QtCore.QSize(size.width() + self.EXTRA_COL_WIDTH, height) - - def paint(self, painter, option, index): - if isinstance(self.parent(), QtWidgets.QTableView): - # Disable rendering of selection highlight for TableViews, they handle - # it themselves with the background colour data: - option.state &= ~(QtWidgets.QStyle.State_Selected) - QtWidgets.QStyledItemDelegate.paint(self, painter, option, index) - if index.column() > 0: - painter.setPen(self._pen) - painter.drawLine(option.rect.topLeft(), option.rect.bottomLeft()) - - def eventFilter(self, obj, event): - """Filter events before they get to the editor, so that editing is ended when - the user presses tab, shift-tab or enter (which otherwise would not end editing - in a QTextEdit).""" - if event.type() == QtCore.QEvent.KeyPress: - if event.key() in [QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return]: - # Allow shift-enter - if not event.modifiers() & QtCore.Qt.ShiftModifier: - self.commitData.emit(obj) - self.closeEditor.emit(obj) - return True - elif event.key() == QtCore.Qt.Key_Tab: - self.commitData.emit(obj) - self.closeEditor.emit(obj, QtWidgets.QStyledItemDelegate.EditNextItem) - return True - elif event.key() == QtCore.Qt.Key_Backtab: - self.commitData.emit(obj) - self.closeEditor.emit(obj, QtWidgets.QStyledItemDelegate.EditPreviousItem) - return True - return QtWidgets.QStyledItemDelegate.eventFilter(self, obj, event) - - def createEditor(self, parent, option, index): - return Editor(parent) - - def setEditorData(self, editor, index): - editor.setPlainText(index.data()) - font = index.data(QtCore.Qt.FontRole) - default_font = qapplication.font(self.parent()) - if font is None: - font = default_font - font.setPointSize(default_font.pointSize()) - editor.setFont(font) - font_height = QtGui.QFontMetrics(font).height() - padding = (self.MIN_ROW_HEIGHT - font_height) / 2 - 1 - editor.document().setDocumentMargin(padding) - editor.selectAll() - - def setModelData(self, editor, model, index): - model.setData(index, editor.toPlainText()) - -class RunmanagerColors(object): - """Singleton class that globally defines various colors for the globals view - - Colors are saved to class attributes as hex strings. - Available colors are: - - :ivar COLOR_HIGHLIGHT: Item selection highlight color, fixed to semitransparent blue - :ivar COLOR_ERROR: Item has runtime error color - :ivar COLOR_OK: Item is normal - :ivar COLOR_BOOL_ON: Item is bool = True - :ivar COLOR_BOOL_OFF: Item is bool = False - """ - _instance = None - - def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super().__new__(cls) - - return cls._instance - - def __init__(self): - - if not hasattr(self, '_initialized'): - self.logger = logging.getLogger('runmanager') - - # init colors - self.update_colors_from_scheme() - - # common color definitions - self.COLOR_HIGHLIGHT = "#40308CC6" # semitransparent blue - - self._initialized = True - - def check_if_light(self): - """Helper method that return current light/dark theme state - - If using PyQt5, always returns True. - Otherwise, state is taken from `QGuiApplication` which follows OS. - """ - - # pyqt5 defaults to light and doesn't have styleHints.colorScheme() - if QT_ENV.lower() == 'pyqt5': - return True - - style_hints = QtGui.QGuiApplication.styleHints() - if style_hints.colorScheme() == QtCore.Qt.ColorScheme.Dark: - return False - else: - return True - - def update_colors_from_scheme(self): - """Method updates colors depending on current color scheme""" - - if self.check_if_light(): - self.logger.info('Setting custom light theme colors') - # use light mode colors for GroupTabs - self.COLOR_ERROR = '#F79494' # light red - self.COLOR_OK = '#A5F7C6' # light green - self.COLOR_BOOL_ON = '#63F731' # bright green - self.COLOR_BOOL_OFF = '#608060' # dark green - else: - self.logger.info('Setting custom dark theme colors') - # use dark mode colors for GroupTabs - self.COLOR_ERROR = "#BC0000" # red - self.COLOR_OK = "#2F4C00" # green - self.COLOR_BOOL_ON = "#29A300" # bright green - self.COLOR_BOOL_OFF = "#003900" # dark green - - class GroupTab(object): GLOBALS_COL_DELETE = 0 GLOBALS_COL_NAME = 1 @@ -864,7 +257,7 @@ def set_tab_icon(self, icon_string): self.tabWidget.setTabIcon(index, icon) def populate_model(self): - globals = runmanager.get_globals({self.group_name: self.globals_file})[self.group_name] + globals = group_manager.get_globals({self.group_name: self.globals_file})[self.group_name] for name, (value, units, expansion) in globals.items(): row = self.make_global_row(name, value, units, expansion) self.globals_model.appendRow(row) @@ -1148,7 +541,7 @@ def new_global(self, global_name): item = self.get_global_item_by_name(global_name, self.GLOBALS_COL_NAME, previous_name=self.GLOBALS_DUMMY_ROW_TEXT) try: - runmanager.new_global(self.globals_file, self.group_name, global_name) + group_manager.new_global(self.globals_file, self.group_name, global_name) except Exception as e: error_dialog(str(e)) else: @@ -1175,7 +568,7 @@ def rename_global(self, previous_global_name, new_global_name): item = self.get_global_item_by_name(new_global_name, self.GLOBALS_COL_NAME, previous_name=previous_global_name) try: - runmanager.rename_global(self.globals_file, self.group_name, previous_global_name, new_global_name) + group_manager.rename_global(self.globals_file, self.group_name, previous_global_name, new_global_name) except Exception as e: error_dialog(str(e)) # Set the item text back to the old name, since the rename failed: @@ -1219,7 +612,7 @@ def change_global_value(self, global_name, previous_value, new_value, interactiv def complete_change_global_value(self, global_name, previous_value, new_value, item, previous_background, previous_icon, interactive=True): try: - runmanager.set_value(self.globals_file, self.group_name, global_name, new_value) + group_manager.set_value(self.globals_file, self.group_name, global_name, new_value) except Exception as e: if interactive: error_dialog(str(e)) @@ -1255,7 +648,7 @@ def change_global_units(self, global_name, previous_units, new_units): (self.globals_file, self.group_name, global_name, previous_units, new_units)) item = self.get_global_item_by_name(global_name, self.GLOBALS_COL_UNITS) try: - runmanager.set_units(self.globals_file, self.group_name, global_name, new_units) + group_manager.set_units(self.globals_file, self.group_name, global_name, new_units) except Exception as e: error_dialog(str(e)) # Set the item text back to the old units, since the change failed: @@ -1272,7 +665,7 @@ def change_global_expansion(self, global_name, previous_expansion, new_expansion (self.globals_file, self.group_name, global_name, previous_expansion, new_expansion)) item = self.get_global_item_by_name(global_name, self.GLOBALS_COL_EXPANSION) try: - runmanager.set_expansion(self.globals_file, self.group_name, global_name, new_expansion) + group_manager.set_expansion(self.globals_file, self.group_name, global_name, new_expansion) except Exception as e: error_dialog(str(e)) # Set the item text back to the old units, since the change failed: @@ -1349,7 +742,7 @@ def delete_global(self, global_name, confirm=True): if confirm: if not question_dialog("Delete the global '%s'?" % global_name): return - runmanager.delete_global(self.globals_file, self.group_name, global_name) + group_manager.delete_global(self.globals_file, self.group_name, global_name) # Find the entry for this global in self.globals_model and remove it: name_item = self.get_global_item_by_name(global_name, self.GLOBALS_COL_NAME) self.globals_model.removeRow(name_item.row()) @@ -2323,7 +1716,7 @@ def on_new_globals_file_clicked(self): # Save the containing folder for use next time we open the dialog box: self.last_opened_globals_folder = os.path.dirname(globals_file) # Create the new file and open it: - runmanager.new_globals_file(globals_file) + group_manager.new_globals_file(globals_file) self.open_globals_file(globals_file) def on_diff_globals_file_clicked(self): @@ -2348,14 +1741,14 @@ def on_diff_globals_file_clicked(self): return # Get file's globals groups - other_groups = runmanager.get_all_groups(globals_file) + other_groups = group_manager.get_all_groups(globals_file) # Display the output tab so the user can see the output: self.ui.tabWidget.setCurrentWidget(self.ui.tab_output) self.output_box.output('Globals diff with:\n%s\n\n' % globals_file) # Do the globals diff - globals_diff_table = runmanager.globals_diff_groups(active_groups, other_groups) + globals_diff_table = differ.globals_diff_groups(active_groups, other_groups) self.output_box.output(globals_diff_table) self.output_box.output('Ready.\n\n') @@ -2590,7 +1983,7 @@ def get_default_output_folder(self): current_labscript_file = self.ui.lineEdit_labscript_file.text() if not current_labscript_file: return '' - _, default_output_folder, _ = runmanager.new_sequence_details( + _, default_output_folder, _ = compiler.new_sequence_details( current_labscript_file, config=self.exp_config, increment_sequence_index=False, @@ -2862,7 +2255,7 @@ def open_globals_file(self, globals_file): return # Get the groups: - groups = runmanager.get_grouplist(globals_file) + groups = group_manager.get_grouplist(globals_file) # Add the parent row: file_name_item = QtGui.QStandardItem(globals_file) file_name_item.setEditable(False) @@ -2993,7 +2386,7 @@ def copy_group(self, source_globals_file, source_group_name, dest_globals_file=N if delete_source_group and source_globals_file == dest_globals_file: return try: - dest_group_name = runmanager.copy_group(source_globals_file, source_group_name, dest_globals_file, delete_source_group) + dest_group_name = group_manager.copy_group(source_globals_file, source_group_name, dest_globals_file, delete_source_group) except Exception as e: error_dialog(str(e)) else: @@ -3031,7 +2424,7 @@ def new_group(self, globals_file, group_name): item = self.get_group_item_by_name(globals_file, group_name, self.GROUPS_COL_NAME, previous_name=self.GROUPS_DUMMY_ROW_TEXT) try: - runmanager.new_group(globals_file, group_name) + group_manager.new_group(globals_file, group_name) except Exception as e: error_dialog(str(e)) else: @@ -3076,7 +2469,7 @@ def rename_group(self, globals_file, previous_group_name, new_group_name): item = self.get_group_item_by_name(globals_file, new_group_name, self.GROUPS_COL_NAME, previous_name=previous_group_name) try: - runmanager.rename_group(globals_file, previous_group_name, new_group_name) + group_manager.rename_group(globals_file, previous_group_name, new_group_name) except Exception as e: error_dialog(str(e)) # Set the item text back to the old name, since the rename failed: @@ -3109,7 +2502,7 @@ def delete_group(self, globals_file, group_name, confirm=True): group_tab = self.currently_open_groups.get((globals_file, group_name)) if group_tab is not None: self.close_group(globals_file, group_name) - runmanager.delete_group(globals_file, group_name) + group_manager.delete_group(globals_file, group_name) # Find the entry for this group in self.groups_model and remove it: name_item = self.get_group_item_by_name(globals_file, group_name, self.GROUPS_COL_NAME) name_item.parent().removeRow(name_item.row()) @@ -3399,15 +2792,15 @@ def compile_loop(self): continue def parse_globals(self, active_groups, raise_exceptions=True, expand_globals=True, expansion_order = None, return_dimensions = False): - sequence_globals = runmanager.get_globals(active_groups) + sequence_globals = group_manager.get_globals(active_groups) #logger.info('got sequence globals') - evaled_globals, global_hierarchy, expansions = runmanager.evaluate_globals(sequence_globals, raise_exceptions) + evaled_globals, global_hierarchy, expansions = group_manager.evaluate_globals(sequence_globals, raise_exceptions) #logger.info('evaluated sequence globals') if expand_globals: if return_dimensions: - shots, dimensions = runmanager.expand_globals(sequence_globals, evaled_globals, expansion_order, return_dimensions=return_dimensions) + shots, dimensions = compiler.expand_globals(sequence_globals, evaled_globals, expansion_order, return_dimensions=return_dimensions) else: - shots = runmanager.expand_globals(sequence_globals, evaled_globals, expansion_order) + shots = compiler.expand_globals(sequence_globals, evaled_globals, expansion_order) else: shots = [] dimensions = {} @@ -3443,7 +2836,7 @@ def guess_expansion_modes(self, active_groups, evaled_globals, global_hierarchy, # Let ExpansionErrors through through, as they occur # when the user has changed the value without changing # the expansion type: - if isinstance(value, runmanager.ExpansionError): + if isinstance(value, ExpansionError): continue return False # Did the guessed expansion type for any of the globals change? @@ -3474,8 +2867,8 @@ def guess_expansion_modes(self, active_groups, evaled_globals, global_hierarchy, else: previous_value = 0 - new_guess = runmanager.guess_expansion_type(new_value) - previous_guess = runmanager.guess_expansion_type(previous_value) + new_guess = group_manager.guess_expansion_type(new_value) + previous_guess = group_manager.guess_expansion_type(previous_value) if new_guess == 'outer': expansion_types[global_name] = {'previous_guess': previous_guess, @@ -3485,7 +2878,7 @@ def guess_expansion_modes(self, active_groups, evaled_globals, global_hierarchy, } elif new_guess != previous_guess: filename = active_groups[group_name] - runmanager.set_expansion(filename, group_name, global_name, new_guess) + group_manager.set_expansion(filename, group_name, global_name, new_guess) expansions[global_name] = new_guess expansion_types_changed = True @@ -3534,7 +2927,7 @@ def set_expansion_type_guess(expansion_types, expansions, global_name, expansion # we have a global that does not depend on anything that has an # expansion type of 'outer' if (not global_depends_on_global_with_outer_product(global_name, global_hierarchy, expansions) - and not isinstance(expansion_types[global_name]['value'], runmanager.ExpansionError)): + and not isinstance(expansion_types[global_name]['value'], ExpansionError)): current_dependencies = find_dependencies(global_name, global_hierarchy, expansion_types) # if this global has other globals that use it, then add them @@ -3548,7 +2941,7 @@ def set_expansion_type_guess(expansion_types, expansions, global_name, expansion for global_name in sorted(self.previous_expansion_types): if (not global_depends_on_global_with_outer_product( global_name, self.previous_global_hierarchy, self.previous_expansions) - and not isinstance(self.previous_expansion_types[global_name]['value'], runmanager.ExpansionError)): + and not isinstance(self.previous_expansion_types[global_name]['value'], ExpansionError)): old_dependencies = find_dependencies(global_name, self.previous_global_hierarchy, self.previous_expansion_types) # if this global has other globals that use it, then add them # all to a zip group with the name of this global @@ -3562,7 +2955,7 @@ def set_expansion_type_guess(expansion_types, expansions, global_name, expansion for global_name, guesses in expansion_types.items(): if guesses['new_guess'] != guesses['previous_guess']: filename = active_groups[guesses['group_name']] - runmanager.set_expansion( + group_manager.set_expansion( filename, str(guesses['group_name']), str(global_name), str(guesses['new_guess'])) expansions[global_name] = guesses['new_guess'] expansion_types_changed = True @@ -3576,7 +2969,7 @@ def set_expansion_type_guess(expansion_types, expansions, global_name, expansion iter(evaled_globals[group_name][global_name]) except Exception: filename = active_groups[group_name] - runmanager.set_expansion(filename, group_name, global_name, '') + group_manager.set_expansion(filename, group_name, global_name, '') expansion_types_changed = True self.previous_evaled_globals = evaled_globals @@ -3587,7 +2980,7 @@ def set_expansion_type_guess(expansion_types, expansions, global_name, expansion return expansion_types_changed def make_h5_files(self, labscript_file, output_folder, sequence_globals, shots, shuffle): - sequence_attrs, default_output_dir, filename_prefix = runmanager.new_sequence_details( + sequence_attrs, default_output_dir, filename_prefix = compiler.new_sequence_details( labscript_file, config=self.exp_config, increment_sequence_index=True ) if output_folder == self.previous_default_output_folder: @@ -3597,7 +2990,7 @@ def make_h5_files(self, labscript_file, output_folder, sequence_globals, shots, # from the UI may be out of date since we only update it once a second. output_folder = default_output_dir self.check_output_folder_update() - run_files = runmanager.make_run_files( + run_files = compiler.make_run_files( output_folder, sequence_globals, shots, @@ -3661,22 +3054,23 @@ def send_to_runviewer(self, run_file): class RemoteServer(ZMQServer): - def __init__(self): - port = app.exp_config.getint( + def __init__(self, app): + self.app = app + port = self.app.exp_config.getint( 'ports', 'runmanager', fallback=runmanager.remote.DEFAULT_PORT ) ZMQServer.__init__(self, port=port) def handle_get_globals(self, raw=False): - active_groups = inmain(app.get_active_groups, interactive=False) - sequence_globals = runmanager.get_globals(active_groups) + active_groups = inmain(self.app.get_active_groups, interactive=False) + sequence_globals = group_manager.get_globals(active_groups) all_globals = {} if raw: for group_globals in sequence_globals.values(): values_only = {name: val for name, (val, _, _) in group_globals.items()} all_globals.update(values_only) else: - evaled_globals, global_hierarchy, expansions = runmanager.evaluate_globals( + evaled_globals, global_hierarchy, expansions = group_manager.evaluate_globals( sequence_globals, raise_exceptions=False ) for group_globals in evaled_globals.values(): @@ -3685,8 +3079,8 @@ def handle_get_globals(self, raw=False): @inmain_decorator() def handle_set_globals(self, globals, raw=False): - active_groups = app.get_active_groups(interactive=False) - sequence_globals = runmanager.get_globals(active_groups) + active_groups = self.app.get_active_groups(interactive=False) + sequence_globals = group_manager.get_globals(active_groups) try: for global_name, new_value in globals.items(): # Unless raw=True, convert to str representation for saving to the GUI @@ -3714,7 +3108,7 @@ def handle_set_globals(self, globals, raw=False): # Append expression-final comments in the previous expression to # the new one: - comments = runmanager.find_comments(previous_value) + comments = tokenizer.find_comments(previous_value) if comments: # Only the final comment comment_start, comment_end = comments[-1] @@ -3723,12 +3117,12 @@ def handle_set_globals(self, globals, raw=False): new_value += previous_value[comment_start:comment_end] try: # Is the group open? - group_tab = app.currently_open_groups[ + group_tab = self.app.currently_open_groups[ globals_file, group_name ] except KeyError: # Group is not open. Change the global value on disk: - runmanager.set_value( + group_manager.set_value( globals_file, group_name, global_name, new_value ) else: @@ -3748,84 +3142,84 @@ def handle_set_globals(self, globals, raw=False): # Trigger preparsing of globals to occur so that changes in globals not in # open tabs are reflected in the GUI, such as n_shots, errors on other # globals that depend on them, etc. - app.globals_changed() + self.app.globals_changed() def handle_engage(self): - app.wait_until_preparse_complete() - inmain(app.on_engage_clicked) + self.app.wait_until_preparse_complete() + inmain(self.app.on_engage_clicked) @inmain_decorator() def handle_abort(self): - app.on_abort_clicked() + self.app.on_abort_clicked() @inmain_decorator() def handle_get_run_shots(self): - return app.ui.checkBox_run_shots.isChecked() + return self.app.ui.checkBox_run_shots.isChecked() @inmain_decorator() def handle_set_run_shots(self, value): - app.ui.checkBox_run_shots.setChecked(value) + self.app.ui.checkBox_run_shots.setChecked(value) @inmain_decorator() def handle_get_view_shots(self): - return app.ui.checkBox_view_shots.isChecked() + return self.app.ui.checkBox_view_shots.isChecked() @inmain_decorator() def handle_set_view_shots(self, value): - app.ui.checkBox_view_shots.setChecked(value) + self.app.ui.checkBox_view_shots.setChecked(value) @inmain_decorator() def handle_get_shuffle(self): - return app.ui.pushButton_shuffle.isChecked() + return self.app.ui.pushButton_shuffle.isChecked() @inmain_decorator() def handle_set_shuffle(self, value): - app.ui.pushButton_shuffle.setChecked(value) + self.app.ui.pushButton_shuffle.setChecked(value) def handle_n_shots(self): # Wait until any current preparsing is done, to ensure this is not racy w.r.t # previous remote calls: - app.wait_until_preparse_complete() - return app.n_shots + self.app.wait_until_preparse_complete() + return self.app.n_shots @inmain_decorator() def handle_get_labscript_file(self): - labscript_file = app.ui.lineEdit_labscript_file.text() + labscript_file = self.app.ui.lineEdit_labscript_file.text() return os.path.abspath(labscript_file) @inmain_decorator() def handle_set_labscript_file(self, value): labscript_file = os.path.abspath(value) - app.ui.lineEdit_labscript_file.setText(labscript_file) + self.app.ui.lineEdit_labscript_file.setText(labscript_file) @inmain_decorator() def handle_get_shot_output_folder(self): - shot_output_folder = app.ui.lineEdit_shot_output_folder.text() + shot_output_folder = self.app.ui.lineEdit_shot_output_folder.text() return os.path.abspath(shot_output_folder) @inmain_decorator() def handle_set_shot_output_folder(self, value): shot_output_folder = os.path.abspath(value) - app.ui.lineEdit_shot_output_folder.setText(shot_output_folder) + self.app.ui.lineEdit_shot_output_folder.setText(shot_output_folder) def handle_error_in_globals(self): try: # This will raise an exception if there are multiple active groups of the # same name: - active_groups = inmain(app.get_active_groups, interactive=False) - sequence_globals = runmanager.get_globals(active_groups) + active_groups = inmain(self.app.get_active_groups, interactive=False) + sequence_globals = group_manager.get_globals(active_groups) # This will raise an exception if any of the globals can't be evaluated: - runmanager.evaluate_globals(sequence_globals, raise_exceptions=True) + group_manager.evaluate_globals(sequence_globals, raise_exceptions=True) except Exception: return True return False def handle_is_output_folder_default(self): - return not app.non_default_folder + return not self.app.non_default_folder @inmain_decorator() def handle_reset_shot_output_folder(self): - app.on_reset_shot_output_folder_clicked(None) + self.app.on_reset_shot_output_folder_clicked(None) def handler(self, request_data): cmd, args, kwargs = request_data @@ -3851,7 +3245,7 @@ def handler(self, request_data): qapplication.setAttribute(QtCore.Qt.AA_DontShowIconsInMenus, False) app = RunManager() splash.update_text('Starting remote server') - remote_server = RemoteServer() + remote_server = RemoteServer(app) splash.hide() # Let the interpreter run every 500ms so it sees Ctrl-C interrupts: diff --git a/runmanager/compiler.py b/runmanager/compiler.py new file mode 100644 index 0000000..c74be03 --- /dev/null +++ b/runmanager/compiler.py @@ -0,0 +1,358 @@ +##################################################################### +# # +# __main__.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the program runmanager, in the labscript # +# suite (see http://labscriptsuite.org), and is licensed under the # +# Simplified BSD License. See the license.txt file in the root of # +# the project for the full license. # +# # +##################################################################### +"""Functions for managing compilation of labscript shots. + +Actual compilation takes place in batch_compiler.py. +These functions prepare the sequence and globals. +""" + +import os +import random +import subprocess +import threading +import traceback +import datetime +import errno +import json + +import h5py +import numpy as np + +from labscript_utils.ls_zprocess import ProcessTree, zmq_push_multipart +from labscript_utils.labconfig import LabConfig +process_tree = ProcessTree.instance() + +from runmanager.expander import expand_globals + +import runmanager.group_manager as group_manager +import runmanager.evaluator as evaluator +import runmanager.expander as expander + +def next_sequence_index(shot_basedir, dt, increment=True): + """Return the next sequence index for sequences in the given base directory (i.e. + /) and the date of the given datetime + object, and increment the sequence index atomically on disk if increment=True. If + not setting increment=True, then the result is indicative only and may be used by + other code at any time. One must increment the sequence index prior to use.""" + from labscript_utils.ls_zprocess import Lock + from labscript_utils.shared_drive import path_to_agnostic + + DATE_FORMAT = '%Y-%m-%d' + # The file where we store the next sequence index on disk: + sequence_index_file = os.path.join(shot_basedir, '.next_sequence_index') + # Open with zlock to prevent race conditions with other code: + with Lock(path_to_agnostic(sequence_index_file), read_only=not increment): + try: + with open(sequence_index_file) as f: + datestr, sequence_index = json.load(f) + if datestr != dt.strftime(DATE_FORMAT): + # New day, start from zero again: + sequence_index = 0 + except (OSError, IOError) as exc: + if exc.errno != errno.ENOENT: + raise + # File doesn't exist yet, start from zero + sequence_index = 0 + if increment: + # Write the new file with the incremented sequence index + os.makedirs(os.path.dirname(sequence_index_file), exist_ok=True) + with open(sequence_index_file, 'w') as f: + json.dump([dt.strftime(DATE_FORMAT), sequence_index + 1], f) + return sequence_index + + +def new_sequence_details(script_path, config=None, increment_sequence_index=True): + """Generate the details for a new sequence: the toplevel attrs sequence_date, + sequence_index, sequence_id; and the the output directory and filename prefix for + the shot files, according to labconfig settings. If increment_sequence_index=True, + then we are claiming the resulting sequence index for use such that it cannot be + used by anyone else. This should be done if the sequence details are immediately + about to be used to compile a sequence. Otherwise, set increment_sequence_index to + False, but in that case the results are indicative only and one should call this + function again with increment_sequence_index=True before compiling the sequence, as + otherwise the sequence_index may be used by other code in the meantime.""" + if config is None: + config = LabConfig() + script_basename = os.path.splitext(os.path.basename(script_path))[0] + shot_storage = config.get('DEFAULT', 'experiment_shot_storage') + shot_basedir = os.path.join(shot_storage, script_basename) + now = datetime.datetime.now() + sequence_timestamp = now.strftime('%Y%m%dT%H%M%S') + + # Toplevel attributes to be saved to the shot files: + sequence_date = now.strftime('%Y-%m-%d') + sequence_id = sequence_timestamp + '_' + script_basename + sequence_index = next_sequence_index(shot_basedir, now, increment_sequence_index) + + sequence_attrs = { + 'script_basename': script_basename, + 'sequence_date': sequence_date, + 'sequence_index': sequence_index, + 'sequence_id': sequence_id, + } + + # Compute the output directory based on labconfig settings: + try: + subdir_format = config.get('runmanager', 'output_folder_format') + except (LabConfig.NoOptionError, LabConfig.NoSectionError): + subdir_format = os.path.join('%Y', '%m', '%d', '{sequence_index:05d}') + + # Format the output directory according to the current timestamp, sequence index and + # sequence_timestamp, if present in the format string: + subdir = now.strftime(subdir_format).format( + sequence_index=sequence_index, sequence_timestamp=sequence_timestamp + ) + shot_output_dir = os.path.join(shot_basedir, subdir) + + # Compute the shot filename prefix according to labconfig settings: + try: + filename_prefix_format = config.get('runmanager', 'filename_prefix_format') + except (LabConfig.NoOptionError, LabConfig.NoSectionError): + # Default, for backward compatibility: + filename_prefix_format = '{sequence_timestamp}_{script_basename}' + # Format the filename prefix according to the current timestamp, sequence index, + # sequence_timestamp, and script_basename, if present in the format string: + filename_prefix = now.strftime(filename_prefix_format).format( + sequence_index=sequence_index, + sequence_timestamp=sequence_timestamp, + script_basename=script_basename, + ) + + return sequence_attrs, shot_output_dir, filename_prefix + + +def make_run_files( + output_folder, + sequence_globals, + shots, + sequence_attrs, + filename_prefix, + shuffle=False, +): + """Does what it says. sequence_globals and shots are of the datatypes returned by + get_globals and get_shots, one is a nested dictionary with string values, and the + other a flat dictionary. sequence_attrs is a dict of the attributes pertaining to + this sequence to be initially set at the top-level group of the h5 file, as returned + by new_sequence_details. output_folder and filename_prefix determine the directory + shot files will be output to, as well as their filenames (this function will + generate filenames with the shot number and .h5 extension appended to + filename_prefix). Sensible defaults for these are also returned by + new_sequence_details(), so preferably these should be used. + + Shuffle will randomise the order that the run files are generated in with respect to + which element of shots they come from. This function returns a *generator*. The run + files are not actually created until you loop over this generator (which gives you + the filepaths). This is useful for not having to clean up as many unused files in + the event of failed compilation of labscripts. If you want all the run files to be + created at some point, simply convert the returned generator to a list. The + filenames the run files are given is simply the sequence_id with increasing integers + appended.""" + basename = os.path.join(output_folder, filename_prefix) + nruns = len(shots) + ndigits = int(np.ceil(np.log10(nruns))) + if shuffle: + random.shuffle(shots) + for i, shot_globals in enumerate(shots): + runfilename = ('%s_%0' + str(ndigits) + 'd.h5') % (basename, i) + make_single_run_file( + runfilename, sequence_globals, shot_globals, sequence_attrs, i, nruns + ) + yield runfilename + + +def make_single_run_file(filename, sequenceglobals, runglobals, sequence_attrs, run_no, n_runs): + """Does what it says. runglobals is a dict of this run's globals, the format being + the same as that of one element of the list returned by expand_globals. + sequence_globals is a nested dictionary of the type returned by get_globals. + sequence_attrs is a dict of attributes pertaining to this sequence, as returned by + new_sequence_details. run_no and n_runs must be provided, if this run file is part + of a sequence, then they should reflect how many run files are being generated in + this sequence, all of which must have identical sequence_attrs.""" + os.makedirs(os.path.dirname(filename), exist_ok=True) + with h5py.File(filename, 'w') as f: + f.attrs.update(sequence_attrs) + f.attrs['run number'] = run_no + f.attrs['n_runs'] = n_runs + f.create_group('globals') + if sequenceglobals is not None: + for groupname, groupvars in sequenceglobals.items(): + group = f['globals'].create_group(groupname) + unitsgroup = group.create_group('units') + expansiongroup = group.create_group('expansion') + for name, (value, units, expansion) in groupvars.items(): + group.attrs[name] = value + unitsgroup.attrs[name] = units + expansiongroup.attrs[name] = expansion + for name, value in runglobals.items(): + if value is None: + # Store it as a null object reference: + value = h5py.Reference() + try: + f['globals'].attrs[name] = value + except Exception as e: + message = ('Global %s cannot be saved as an hdf5 attribute. ' % name + + 'Globals can only have relatively simple datatypes, with no nested structures. ' + + 'Original error was:\n' + + '%s: %s' % (e.__class__.__name__, str(e))) + raise ValueError(message) + + +def make_run_file_from_globals_files(labscript_file, globals_files, output_path, config=None): + """Creates a run file output_path, using all the globals from globals_files. Uses + labscript_file to determine the sequence_attrs only""" + groups = group_manager.get_all_groups(globals_files) + sequence_globals = group_manager.get_globals(groups) + evaled_globals, global_hierarchy, expansions = evaluator.evaluate_globals(sequence_globals) + shots = expander.expand_globals(sequence_globals, evaled_globals) + if len(shots) > 1: + scanning_globals = [] + for global_name in expansions: + if expansions[global_name]: + scanning_globals.append(global_name) + raise ValueError('Cannot compile to a single run file: The following globals are a sequence: ' + + ', '.join(scanning_globals)) + + sequence_attrs, _, _ = new_sequence_details( + labscript_file, config=config, increment_sequence_index=True + ) + make_single_run_file(output_path, sequence_globals, shots[0], sequence_attrs, 1, 1) + + +def compile_labscript_async(labscript_file, run_file, + stream_port=None, done_callback=None): + """Compiles labscript_file with run_file. + + This function is designed to be called in a thread. + The stdout and stderr from the compilation will be shovelled into + stream_port via zmq push as it spews forth, and when compilation is complete, + done_callback will be called with a boolean argument indicating success. Note that + the zmq communication will be encrypted, or not, according to security settings in + labconfig. If you want to receive the data on a zmq socket, do so using a PULL + socket created from a :class:`labscript_utils.ls_zprocess.Context`, or using a + :class:`labscript_utils.ls_zprocess.ZMQServer`. These subclasses will also be configured + with the appropriate security settings and will be able to receive the messages. + + Args: + labscript_file (str): Path to labscript file to be compiled + run_file (str): Path to h5 file where compilation output is stored. + This file must already exist with proper globals initialization. + See :func:`new_globals_file` for details. + stream_port (zmq.socket, optional): ZMQ socket to push stdout and stderr. + If None, defaults to calling process stdout/stderr. Default is None. + done_callback (function, optional): Callback function run when compilation finishes. + Takes a single boolean argument marking compilation success or failure. + If None, callback is skipped. Default is None. + """ + compiler_path = os.path.join(os.path.dirname(__file__), 'batch_compiler.py') + to_child, from_child, child = process_tree.subprocess( + compiler_path, output_redirection_port=stream_port + ) + to_child.put(['compile', [labscript_file, run_file]]) + while True: + signal, data = from_child.get() + if signal == 'done': + success = data + to_child.put(['quit', None]) + child.communicate() + if done_callback is not None: + done_callback(success) + break + else: + raise RuntimeError((signal, data)) + + +def compile_multishot_async(labscript_file, run_files, + stream_port=None, done_callback=None): + """Compiles labscript_file with multiple run_files (ie globals). + + This function is designed to be called in a thread. + The stdout and stderr from the compilation will be shovelled into + stream_port via zmq push as it spews forth, and when each compilation is complete, + done_callback will be called with a boolean argument indicating success. Compilation + will stop after the first failure. If you want to receive the data on a zmq socket, + do so using a PULL socket created from a :class:`labscript_utils.ls_zprocess.Context`, or + using a :class:`labscript_utils.ls_zprocess.ZMQServer`. These subclasses will also be + configured with the appropriate security settings and will be able to receive the + messages. + + Args: + labscript_file (str): Path to labscript file to be compiled + run_files (list of str): Paths to h5 file where compilation output is stored. + These files must already exist with proper globals initialization. + stream_port (zmq.socket, optional): ZMQ socket to push stdout and stderr. + If None, defaults to calling process stdout/stderr. Default is None. + done_callback (function, optional): Callback function run when compilation finishes. + Takes a single boolean argument marking compilation success or failure. + If None, callback is skipped. Default is None. + """ + compiler_path = os.path.join(os.path.dirname(__file__), 'batch_compiler.py') + to_child, from_child, child = process_tree.subprocess( + compiler_path, output_redirection_port=stream_port + ) + try: + for run_file in run_files: + to_child.put(['compile', [labscript_file, run_file]]) + while True: + signal, data = from_child.get() + if signal == 'done': + success = data + if done_callback is not None: + done_callback(data) + break + if not success: + break + except Exception: + error = traceback.format_exc() + zmq_push_multipart(stream_port, data=[b'stderr', error.encode('utf-8')]) + to_child.put(['quit', None]) + child.communicate() + raise + to_child.put(['quit', None]) + child.communicate() + + +def compile_labscript_with_globals_files_async(labscript_file, globals_files, output_path, + stream_port, done_callback): + """Compiles labscript_file with multiple globals files into a directory. + + Instead, stderr and stdout will be put to + stream_port via zmq push in the multipart message format `['stdout','hello, world\\n']` + etc. When compilation is finished, the function done_callback will be called a + boolean argument indicating success or failure. If you want to receive the data on + a zmq socket, do so using a PULL socket created from a + :external:class:`labscript_utils.ls_zprocess.Context`, or using a + :external:class:`labscript_utils.ls_zprocess.ZMQServer`. These subclasses will also be configured with + the appropriate security settings and will be able to receive the messages. + + Args: + labscript_file (str): Path to labscript file to be compiled + globals_files (list of str): Paths to h5 file where globals values to be used are stored. + See :func:`make_run_file_from_globals_files` for details. + output_path (str): Folder to save compiled h5 files to. + stream_port (zmq.socket): ZMQ socket to push stdout and stderr. + done_callback (function): Callback function run when compilation finishes. + Takes a single boolean argument marking compilation success or failure. + """ + try: + make_run_file_from_globals_files(labscript_file, globals_files, output_path) + thread = threading.Thread( + target=compile_labscript_async, args=[labscript_file, output_path, stream_port, done_callback]) + thread.daemon = True + thread.start() + except Exception: + error = traceback.format_exc() + zmq_push_multipart(stream_port, data=[b'stderr', error.encode('utf-8')]) + t = threading.Thread(target=done_callback, args=(False,)) + t.daemon = True + t.start() + diff --git a/runmanager/differ.py b/runmanager/differ.py new file mode 100644 index 0000000..1488c08 --- /dev/null +++ b/runmanager/differ.py @@ -0,0 +1,125 @@ +##################################################################### +# # +# __main__.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the program runmanager, in the labscript # +# suite (see http://labscriptsuite.org), and is licensed under the # +# Simplified BSD License. See the license.txt file in the root of # +# the project for the full license. # +# # +##################################################################### +"""Functions for comparing two globals groups. +""" + +import numpy as np + +from runmanager.evaluator import evaluate_globals +from runmanager.tokenizer import remove_comments_and_tokenify +from runmanager.group_manager import get_all_groups, get_globals + +def flatten_globals(sequence_globals, evaluated=False): + """Flattens the data structure of the globals. If evaluated=False, + saves only the value expression string of the global, not the + units or expansion.""" + flattened_sequence_globals = {} + for globals_group in sequence_globals.values(): + for name, value in globals_group.items(): + if evaluated: + flattened_sequence_globals[name] = value + else: + value_expression, units, expansion = value + flattened_sequence_globals[name] = value_expression + return flattened_sequence_globals + + +def dict_diff(dict1, dict2): + """Return the difference between two dictionaries as a dictionary of key: [val1, val2] pairs. + Keys unique to either dictionary are included as key: [val1, '-'] or key: ['-', val2].""" + diff_keys = [] + common_keys = np.intersect1d(list(dict1.keys()), list(dict2.keys())) + for key in common_keys: + if np.iterable(dict1[key]) or np.iterable(dict2[key]): + if not np.array_equal(dict1[key], dict2[key]): + diff_keys.append(key) + else: + if dict1[key] != dict2[key]: + diff_keys.append(key) + + dict1_unique = [key for key in dict1.keys() if key not in common_keys] + dict2_unique = [key for key in dict2.keys() if key not in common_keys] + + diff = {} + for key in diff_keys: + diff[key] = [dict1[key], dict2[key]] + + for key in dict1_unique: + diff[key] = [dict1[key], '-'] + + for key in dict2_unique: + diff[key] = ['-', dict2[key]] + + return diff + + +def globals_diff_groups(active_groups, other_groups, max_cols=1000, return_string=True): + """Given two sets of globals groups, perform a diff of the raw + and evaluated globals.""" + our_sequence_globals = get_globals(active_groups) + other_sequence_globals = get_globals(other_groups) + + # evaluate globals + our_evaluated_sequence_globals, _, _ = evaluate_globals(our_sequence_globals, raise_exceptions=False) + other_evaluated_sequence_globals, _, _ = evaluate_globals(other_sequence_globals, raise_exceptions=False) + + # flatten globals dictionaries + our_globals = flatten_globals(our_sequence_globals, evaluated=False) + other_globals = flatten_globals(other_sequence_globals, evaluated=False) + our_evaluated_globals = flatten_globals(our_evaluated_sequence_globals, evaluated=True) + other_evaluated_globals = flatten_globals(other_evaluated_sequence_globals, evaluated=True) + + # diff the *evaluated* globals + value_differences = dict_diff(other_evaluated_globals, our_evaluated_globals) + + # We are interested only in displaying globals where *both* the + # evaluated global *and* its unevaluated expression (ignoring comments + # and whitespace) differ. This will minimise false positives where a + # slight change in an expression still leads to the same value, or + # where an object has a poorly defined equality operator that returns + # False even when the two objects are identical. + filtered_differences = {} + for name, (other_value, our_value) in value_differences.items(): + our_expression = our_globals.get(name, '-') + other_expression = other_globals.get(name, '-') + # Strip comments, get tokens so we can diff without being sensitive to comments or whitespace: + our_expression, our_tokens = remove_comments_and_tokenify(our_expression) + other_expression, other_tokens = remove_comments_and_tokenify(other_expression) + if our_tokens != other_tokens: + filtered_differences[name] = [repr(other_value), repr(our_value), other_expression, our_expression] + if filtered_differences: + import pandas as pd + df = pd.DataFrame.from_dict(filtered_differences, 'index') + df = df.sort_index() + df.columns = ['Prev (Eval)', 'Current (Eval)', 'Prev (Raw)', 'Current (Raw)'] + df_string = df.to_string(max_cols=max_cols) + payload = df_string + '\n\n' + else: + payload = 'Evaluated globals are identical to those of selected file.\n' + if return_string: + return payload + else: + print(payload) + return df + + +def globals_diff_shots(file1, file2, max_cols=100): + # Get file's globals groups + active_groups = get_all_groups(file1) + + # Get other file's globals groups + other_groups = get_all_groups(file2) + + print('Globals diff between:\n%s\n%s\n\n' % (file1, file2)) + return globals_diff_groups(active_groups, other_groups, max_cols=max_cols, return_string=False) + diff --git a/runmanager/evaluator.py b/runmanager/evaluator.py new file mode 100644 index 0000000..d27eca7 --- /dev/null +++ b/runmanager/evaluator.py @@ -0,0 +1,162 @@ +##################################################################### +# # +# __main__.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the program runmanager, in the labscript # +# suite (see http://labscriptsuite.org), and is licensed under the # +# Simplified BSD License. See the license.txt file in the root of # +# the project for the full license. # +# # +##################################################################### +"""Functions for evaluating runmanager globals. + +Evaluating converts strings into python expressions. +""" + +import types + +from runmanager.exceptions import ExpansionError + +class TraceDictionary(dict): + + def __init__(self, *args, **kwargs): + self.trace_data = None + dict.__init__(self, *args, **kwargs) + + def start_trace(self): + self.trace_data = [] + + def __getitem__(self, key): + if self.trace_data is not None: + if key not in self.trace_data: + self.trace_data.append(key) + return dict.__getitem__(self, key) + + def stop_trace(self): + trace_data = self.trace_data + self.trace_data = None + return trace_data + +def evaluate_globals(sequence_globals, raise_exceptions=True): + """Takes a dictionary of globals as returned by get_globals. These + globals are unevaluated strings. Evaluates them all in the same + namespace so that the expressions can refer to each other. Iterates + to allow for NameErrors to be resolved by subsequently defined + globals. Throws an exception if this does not result in all errors + going away. The exception contains the messages of all exceptions + which failed to be resolved. If raise_exceptions is False, any + evaluations resulting in an exception will instead return the + exception object in the results dictionary""" + + # Flatten all the groups into one dictionary of {global_name: + # expression} pairs. Also create the group structure of the results + # dict, which has the same structure as sequence_globals: + all_globals = {} + results = {} + expansions = {} + global_hierarchy = {} + # Pre-fill the results dictionary with groups, this is needed for + # storing exceptions in the case of globals with the same name being + # defined in multiple groups (all of them get the exception): + for group_name in sequence_globals: + results[group_name] = {} + multiply_defined_globals = set() + for group_name in sequence_globals: + for global_name in sequence_globals[group_name]: + if global_name in all_globals: + # The same global is defined twice. Either raise an + # exception, or store the exception for each place it is + # defined, depending on whether raise_exceptions is True: + groups_with_same_global = [] + for other_group_name in sequence_globals: + if global_name in sequence_globals[other_group_name]: + groups_with_same_global.append(other_group_name) + exception = ValueError('Global named \'%s\' is defined in multiple active groups:\n ' % global_name + + '\n '.join(groups_with_same_global)) + if raise_exceptions: + raise exception + for other_group_name in groups_with_same_global: + results[other_group_name][global_name] = exception + multiply_defined_globals.add(global_name) + all_globals[global_name], units, expansion = sequence_globals[group_name][global_name] + expansions[global_name] = expansion + + # Do not attempt to evaluate globals which are multiply defined: + for global_name in multiply_defined_globals: + del all_globals[global_name] + + # Eval the expressions in the same namespace as each other: + evaled_globals = {} + # we use a "TraceDictionary" to track which globals another global depends on + sandbox = TraceDictionary() + exec('from pylab import *', sandbox, sandbox) + exec('from runmanager.functions import *', sandbox, sandbox) + globals_to_eval = all_globals.copy() + previous_errors = -1 + while globals_to_eval: + errors = [] + for global_name, expression in globals_to_eval.copy().items(): + # start the trace to determine which globals this global depends on + sandbox.start_trace() + try: + code = compile(expression, '', 'eval') + value = eval(code, sandbox) + # Need to know the length of any generators, convert to tuple: + if isinstance(value, types.GeneratorType): + value = iterator_to_tuple(value) + # Make sure if we're zipping or outer-producting this value, that it can + # be iterated over: + if expansions[global_name] == 'outer': + try: + iter(value) + except Exception as e: + raise ExpansionError(str(e)) + except Exception as e: + # Don't raise, just append the error to a list, we'll display them all later. + errors.append((global_name, e)) + sandbox.stop_trace() + continue + # Put the global into the namespace so other globals can use it: + sandbox[global_name] = value + del globals_to_eval[global_name] + evaled_globals[global_name] = value + + # get the results from the global trace + trace_data = sandbox.stop_trace() + # Only store names of globals (not other functions) + for key in list(trace_data): # copy the list before iterating over it + if key not in all_globals: + trace_data.remove(key) + if trace_data: + global_hierarchy[global_name] = trace_data + + if len(errors) == previous_errors: + # Since some globals may refer to others, we expect maybe + # some NameErrors to have occured. There should be fewer + # NameErrors each iteration of this while loop, as globals + # that are required become defined. If there are not fewer + # errors, then there is something else wrong and we should + # raise it. + if raise_exceptions: + message = 'Error parsing globals:\n' + for global_name, exception in errors: + message += '%s: %s: %s\n' % (global_name, exception.__class__.__name__, str(exception)) + raise Exception(message) + else: + for global_name, exception in errors: + evaled_globals[global_name] = exception + break + previous_errors = len(errors) + + # Assemble results into a dictionary of the same format as sequence_globals: + for group_name in sequence_globals: + for global_name in sequence_globals[group_name]: + # Do not attempt to override exception objects already stored + # as the result of multiply defined globals: + if global_name not in results[group_name]: + results[group_name][global_name] = evaled_globals[global_name] + + return results, global_hierarchy, expansions + diff --git a/runmanager/exceptions.py b/runmanager/exceptions.py new file mode 100644 index 0000000..b8b6add --- /dev/null +++ b/runmanager/exceptions.py @@ -0,0 +1,5 @@ +class ExpansionError(Exception): + + """An exception class so that error handling code can tell when a + parsing exception was caused by a mismatch with the expansion mode""" + pass diff --git a/runmanager/expander.py b/runmanager/expander.py new file mode 100644 index 0000000..de67966 --- /dev/null +++ b/runmanager/expander.py @@ -0,0 +1,126 @@ +##################################################################### +# # +# __main__.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the program runmanager, in the labscript # +# suite (see http://labscriptsuite.org), and is licensed under the # +# Simplified BSD License. See the license.txt file in the root of # +# the project for the full license. # +# # +##################################################################### +"""Functions for expanding iterable globals into a list of shots. +""" + +import itertools +import random + +import numpy as np + +def expand_globals(sequence_globals, evaled_globals, expansion_config = None, return_dimensions = False): + """Expands iterable globals according to their expansion + settings. Creates a number of 'axes' which are to be outer product'ed + together. Some of these axes have only one element, these are globals + that do not vary. Some have a set of globals being zipped together, + iterating in lock-step. Others contain a single global varying + across its values (the globals set to 'outer' expansion). Returns + a list of shots, each element of which is a dictionary for that + shot's globals.""" + + if expansion_config is None: + order = {} + shuffle = {} + else: + order = {k:v['order'] for k,v in expansion_config.items() if 'order' in v} + shuffle = {k:v['shuffle'] for k,v in expansion_config.items() if 'shuffle' in v} + + values = {} + expansions = {} + for group_name in sequence_globals: + for global_name in sequence_globals[group_name]: + expression, units, expansion = sequence_globals[group_name][global_name] + value = evaled_globals[group_name][global_name] + values[global_name] = value + expansions[global_name] = expansion + + # Get a list of the zip keys in use: + zip_keys = set(expansions.values()) + try: + zip_keys.remove('outer') + except KeyError: + pass + + axes = {} + global_names = {} + dimensions = {} + for zip_key in zip_keys: + axis = [] + zip_global_names = [] + for global_name in expansions: + if expansions[global_name] == zip_key: + value = values[global_name] + if isinstance(value, Exception): + continue + if not zip_key: + # Wrap up non-iterating globals (with zip_key = '') in a + # one-element list. When zipped and then outer product'ed, + # this will give us the result we want: + value = [value] + axis.append(value) + zip_global_names.append(global_name) + axis = list(zip(*axis)) + dimensions['zip '+zip_key] = len(axis) + axes['zip '+zip_key] = axis + global_names['zip '+zip_key] = zip_global_names + + # Give each global being outer-product'ed its own axis. It gets + # wrapped up in a list and zipped with itself so that it is in the + # same format as the zipped globals, ready for outer-producting + # together: + for global_name in expansions: + if expansions[global_name] == 'outer': + value = values[global_name] + if isinstance(value, Exception): + continue + axis = [value] + axis = list(zip(*axis)) + dimensions['outer '+global_name] = len(axis) + axes['outer '+global_name] = axis + global_names['outer '+global_name] = [global_name] + + # add any missing items to order and dimensions + for key, value in axes.items(): + if key not in order: + order[key] = -1 + if key not in shuffle: + shuffle[key] = False + if key not in dimensions: + dimensions[key] = 1 + + # shuffle relevant axes + for axis_name, axis_values in axes.items(): + if shuffle[axis_name]: + random.shuffle(axis_values) + + # sort axes and global names by order + axes = [axes.get(key) for key in sorted(order, key=order.get)] + global_names = [global_names.get(key) for key in sorted(order, key=order.get)] + + # flatten the global names + global_names = [global_name for global_list in global_names for global_name in global_list] + + + shots = [] + for axis_values in itertools.product(*axes): + # values here is a tuple of tuples, with the outer list being over + # the axes. We need to flatten it to get our individual values out + # for each global, since we no longer care what axis they are on: + global_values = [value for axis in axis_values for value in axis] + shot_globals = dict(zip(global_names, global_values)) + shots.append(shot_globals) + + if return_dimensions: + return shots, dimensions + else: + return shots diff --git a/runmanager/globals_diff.py b/runmanager/globals_diff.py index 5149fd7..3e6262b 100644 --- a/runmanager/globals_diff.py +++ b/runmanager/globals_diff.py @@ -1,13 +1,13 @@ -"""Script that runs :meth:`runmanager.globals_diff_shots` between two shot files. +"""Script that runs :meth:`runmanager.differ.globals_diff_shots` between two shot files. It is run from the command prompt:: -$ python runmanager.global_diffs(shot1,shot2) +$ python runmanager.differ.global_diffs(shot1,shot2) """ import sys -from runmanager import globals_diff_shots +from runmanager.differ import globals_diff_shots if __name__ == '__main__': diff --git a/runmanager/group_manager.py b/runmanager/group_manager.py new file mode 100644 index 0000000..50006ae --- /dev/null +++ b/runmanager/group_manager.py @@ -0,0 +1,348 @@ +##################################################################### +# # +# __main__.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the program runmanager, in the labscript # +# suite (see http://labscriptsuite.org), and is licensed under the # +# Simplified BSD License. See the license.txt file in the root of # +# the project for the full license. # +# # +##################################################################### +"""Functions for reading, writing, and modifying globals groups. +""" + +import io +import h5py +import numpy as np +import tokenize + +import labscript_utils.shot_utils + +from runmanager.evaluator import evaluate_globals + +def _ensure_str(s): + """convert bytestrings and numpy strings to python strings""" + return s.decode() if isinstance(s, bytes) else str(s) + + +def is_valid_python_identifier(name): + # No whitespace allowed. Do this check here because an actual newline in the source + # is not easily distinguished from a NEWLINE token in the produced tokens, which is + # produced even when there is no newline character in the string. So since we ignore + # NEWLINE later, we must check for it now. + if name != "".join(name.split()): + return False + try: + tokens = list(tokenize.generate_tokens(io.StringIO(name).readline)) + except tokenize.TokenError: + return False + token_types = [ + t[0] for t in tokens if t[0] not in [tokenize.NEWLINE, tokenize.ENDMARKER] + ] + if len(token_types) == 1: + return token_types[0] == tokenize.NAME + return False + + +def is_valid_hdf5_group_name(name): + """Ensure that a string is a valid name for an hdf5 group. + + The names of hdf5 groups may only contain ASCII characters. Furthermore, the + characters "/" and "." are not allowed. + + Args: + name (str): The potential name for an hdf5 group. + + Returns: + bool: Whether or not `name` is a valid name for an hdf5 group. This will + be `True` if it is a valid name or `False` otherwise. + """ + # Ensure only ASCII characters are used. + for char in name: + if ord(char) >= 128: + return False + + # Ensure forbidden ASCII characters are not used. + forbidden_characters = ['.', '/'] + for character in forbidden_characters: + if character in name: + return False + return True + +def new_globals_file(filename): + """Creates a new globals h5 file. + + Creates a 'globals' group at the top level. + If file does not exist, a new h5 file is created. + """ + with h5py.File(filename, 'w') as f: + f.create_group('globals') + +def add_expansion_groups(filename): + """backward compatability, for globals files which don't have + expansion groups. Create them if they don't exist. Guess expansion + settings based on datatypes, if possible.""" + # DEPRECATED + # Don't open in write mode unless we have to: + with h5py.File(filename, 'r') as f: + requires_expansion_group = [] + for groupname in f['globals']: + group = f['globals'][groupname] + if 'expansion' not in group: + requires_expansion_group.append(groupname) + if requires_expansion_group: + group_globalslists = [get_globalslist(filename, groupname) for groupname in requires_expansion_group] + with h5py.File(filename, 'a') as f: + for groupname, globalslist in zip(requires_expansion_group, group_globalslists): + group = f['globals'][groupname] + subgroup = group.create_group('expansion') + # Initialise all expansion settings to blank strings: + for name in globalslist: + subgroup.attrs[name] = '' + groups = {group_name: filename for group_name in get_grouplist(filename)} + sequence_globals = get_globals(groups) + evaled_globals, global_hierarchy, expansions = evaluate_globals(sequence_globals, raise_exceptions=False) + for group_name in evaled_globals: + for global_name in evaled_globals[group_name]: + value = evaled_globals[group_name][global_name] + expansion = guess_expansion_type(value) + set_expansion(filename, group_name, global_name, expansion) + + +def get_grouplist(filename): + # For backward compatability, add 'expansion' settings to this + # globals file, if it doesn't contain any. Guess expansion settings + # if possible. + # DEPRECATED + add_expansion_groups(filename) + with h5py.File(filename, 'r') as f: + grouplist = f['globals'] + # File closes after this function call, so have to + # convert the grouplist generator to a list of strings + # before its file gets dereferenced: + return list(grouplist) + + +def new_group(filename, groupname): + if not is_valid_hdf5_group_name(groupname): + raise ValueError( + 'Invalid group name. Group names must contain only ASCII ' + 'characters and cannot include "/" or ".".' + ) + with h5py.File(filename, 'a') as f: + if groupname in f['globals']: + raise Exception('Can\'t create group: target name already exists.') + group = f['globals'].create_group(groupname) + group.create_group('units') + group.create_group('expansion') + + +def copy_group(source_globals_file, source_groupname, dest_globals_file, delete_source_group=False): + """ This function copies the group source_groupname from source_globals_file + to dest_globals_file and renames the new group so that there is no name + collision. If delete_source_group is False the copyied files have + a suffix '_copy'.""" + with h5py.File(source_globals_file, 'a') as source_f: + # check if group exists + if source_groupname not in source_f['globals']: + raise Exception('Can\'t copy there is no group "{}"!'.format(source_groupname)) + + # Are we coping from one file to another? + if dest_globals_file is not None and source_globals_file != dest_globals_file: + dest_f = h5py.File(dest_globals_file, 'a') # yes -> open dest_globals_file + else: + dest_f = source_f # no -> dest files is source file + + # rename Group until there is no name collisions + i = 0 if not delete_source_group else 1 + dest_groupname = source_groupname + while dest_groupname in dest_f['globals']: + dest_groupname = "{}({})".format(dest_groupname, i) if i > 0 else "{}_copy".format(dest_groupname) + i += 1 + + # copy group + dest_f.copy(source_f['globals'][source_groupname], '/globals/%s' % dest_groupname) + + # close opend file + if dest_f != source_f: + dest_f.close() + + return dest_groupname + + +def rename_group(filename, oldgroupname, newgroupname): + if oldgroupname == newgroupname: + # No rename! + return + if not is_valid_hdf5_group_name(newgroupname): + raise ValueError( + 'Invalid group name. Group names must contain only ASCII ' + 'characters and cannot include "/" or ".".' + ) + with h5py.File(filename, 'a') as f: + if newgroupname in f['globals']: + raise Exception('Can\'t rename group: target name already exists.') + f.copy(f['globals'][oldgroupname], '/globals/%s' % newgroupname) + del f['globals'][oldgroupname] + + +def delete_group(filename, groupname): + with h5py.File(filename, 'a') as f: + del f['globals'][groupname] + + +def get_globalslist(filename, groupname): + with h5py.File(filename, 'r') as f: + group = f['globals'][groupname] + # File closes after this function call, so have to convert + # the attrs to a dict before its file gets dereferenced: + return dict(group.attrs) + + +def new_global(filename, groupname, globalname): + if not is_valid_python_identifier(globalname): + raise ValueError('%s is not a valid Python variable name'%globalname) + with h5py.File(filename, 'a') as f: + group = f['globals'][groupname] + if globalname in group.attrs: + raise Exception('Can\'t create global: target name already exists.') + group.attrs[globalname] = '' + f['globals'][groupname]['units'].attrs[globalname] = '' + f['globals'][groupname]['expansion'].attrs[globalname] = '' + + +def rename_global(filename, groupname, oldglobalname, newglobalname): + if oldglobalname == newglobalname: + # No rename! + return + if not is_valid_python_identifier(newglobalname): + raise ValueError('%s is not a valid Python variable name'%newglobalname) + value = get_value(filename, groupname, oldglobalname) + units = get_units(filename, groupname, oldglobalname) + expansion = get_expansion(filename, groupname, oldglobalname) + with h5py.File(filename, 'a') as f: + group = f['globals'][groupname] + if newglobalname in group.attrs: + raise Exception('Can\'t rename global: target name already exists.') + group.attrs[newglobalname] = value + group['units'].attrs[newglobalname] = units + group['expansion'].attrs[newglobalname] = expansion + del group.attrs[oldglobalname] + del group['units'].attrs[oldglobalname] + del group['expansion'].attrs[oldglobalname] + + +def get_value(filename, groupname, globalname): + with h5py.File(filename, 'r') as f: + value = f['globals'][groupname].attrs[globalname] + # Replace numpy strings with python unicode strings. + # DEPRECATED, for backward compat with old files + value = _ensure_str(value) + return value + + +def set_value(filename, groupname, globalname, value): + with h5py.File(filename, 'a') as f: + f['globals'][groupname].attrs[globalname] = value + + +def get_units(filename, groupname, globalname): + with h5py.File(filename, 'r') as f: + value = f['globals'][groupname]['units'].attrs[globalname] + # Replace numpy strings with python unicode strings. + # DEPRECATED, for backward compat with old files + value = _ensure_str(value) + return value + + +def set_units(filename, groupname, globalname, units): + with h5py.File(filename, 'a') as f: + f['globals'][groupname]['units'].attrs[globalname] = units + + +def get_expansion(filename, groupname, globalname): + with h5py.File(filename, 'r') as f: + value = f['globals'][groupname]['expansion'].attrs[globalname] + # Replace numpy strings with python unicode strings. + # DEPRECATED, for backward compat with old files + value = _ensure_str(value) + return value + + +def set_expansion(filename, groupname, globalname, expansion): + with h5py.File(filename, 'a') as f: + f['globals'][groupname]['expansion'].attrs[globalname] = expansion + + +def delete_global(filename, groupname, globalname): + with h5py.File(filename, 'a') as f: + group = f['globals'][groupname] + del group.attrs[globalname] + + +def guess_expansion_type(value): + if isinstance(value, np.ndarray) or isinstance(value, list): + return u'outer' + else: + return u'' + + +def get_all_groups(h5_files): + """returns a dictionary of group_name: h5_path pairs from a list of h5_files.""" + if isinstance(h5_files, bytes) or isinstance(h5_files, str): + h5_files = [h5_files] + groups = {} + for path in h5_files: + for group_name in get_grouplist(path): + if group_name in groups: + raise ValueError('Error: group %s is defined in both %s and %s. ' % (group_name, groups[group_name], path) + + 'Only uniquely named groups can be used together ' + 'to make a run file.') + groups[group_name] = path + return groups + + +def get_globals(groups): + """Takes a dictionary of group_name: h5_file pairs and pulls the + globals out of the groups in their files. The globals are strings + storing python expressions at this point. All these globals are + packed into a new dictionary, keyed by group_name, where the values + are dictionaries which look like {global_name: (expression, units, expansion), ...}""" + # get a list of filepaths: + filepaths = set(groups.values()) + sequence_globals = {} + for filepath in filepaths: + groups_from_this_file = [g for g, f in groups.items() if f == filepath] + with h5py.File(filepath, 'r') as f: + for group_name in groups_from_this_file: + sequence_globals[group_name] = {} + globals_group = f['globals'][group_name] + values = dict(globals_group.attrs) + units = dict(globals_group['units'].attrs) + expansions = dict(globals_group['expansion'].attrs) + for global_name, value in values.items(): + unit = units[global_name] + expansion = expansions[global_name] + # Replace numpy strings with python unicode strings. + # DEPRECATED, for backward compat with old files + value = _ensure_str(value) + unit = _ensure_str(unit) + expansion = _ensure_str(expansion) + sequence_globals[group_name][global_name] = value, unit, expansion + return sequence_globals + +def get_shot_globals(filepath): + """Returns the evaluated globals for a shot, for use by labscript or lyse. + Simple dictionary access as in dict(h5py.File(filepath).attrs) would be fine + except we want to apply some hacks, so it's best to do that in one place. + + Deprecated: use identical function `labscript_utils.shot_utils.get_shot_globals` + """ + + warnings.warn( + FutureWarning("get_shot_globals has moved to labscript_utils.shot_utils. " + "Please update your code to import it from there.")) + + return labscript_utils.shot_utils.get_shot_globals(filepath) diff --git a/runmanager/tokenizer.py b/runmanager/tokenizer.py new file mode 100644 index 0000000..c1cd8e9 --- /dev/null +++ b/runmanager/tokenizer.py @@ -0,0 +1,74 @@ +##################################################################### +# # +# __main__.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the program runmanager, in the labscript # +# suite (see http://labscriptsuite.org), and is licensed under the # +# Simplified BSD License. See the license.txt file in the root of # +# the project for the full license. # +# # +##################################################################### +"""Functions for parsing global expressions, especially for comments. +""" + +import io +import tokenize + +def find_comments(src): + """Return a list of start and end indices for where comments are in given Python + source. Comments on separate lines with only whitespace in between them are + coalesced. Whitespace preceding a comment is counted as part of the comment.""" + line_start = 0 + comments = [] + tokens = tokenize.generate_tokens(io.StringIO(src).readline) + try: + for token_type, token_value, (_, start), (_, end), _ in tokens: + if token_type == tokenize.COMMENT: + comments.append((line_start + start, line_start + end)) + if token_value == '\n': + line_start += end + except tokenize.TokenError: + pass + # coalesce comments with only whitespace between them: + to_merge = [] + for i, ((start1, end1), (start2, end2)) in enumerate(zip(comments, comments[1:])): + if not src[end1:start2].strip(): + to_merge.append(i) + # Reverse order so deletion doesn't change indices: + for i in reversed(to_merge): + start1, end1 = comments[i] + start2, end2 = comments[i + 1] + comments[i] = (start1, end2) + del comments[i + 1] + # Extend each comment block to the left to include whitespace: + for i, (start, end) in enumerate(comments): + n_whitespace_chars = len(src[:start]) - len(src[:start].rstrip()) + comments[i] = start - n_whitespace_chars, end + # Extend the final comment to the right to include whitespace: + if comments: + start, end = comments[-1] + n_whitespace_chars = len(src[end:]) - len(src[end:].rstrip()) + comments[-1] = (start, end + n_whitespace_chars) + return comments + + +def remove_comments_and_tokenify(src): + """Removes comments from source code, leaving it otherwise intact, + and returns it. Also returns the raw tokens for the code, allowing + comparisons between source to be made without being sensitive to + whitespace.""" + # Remove comments + for (start, end) in reversed(find_comments(src)): + src = src[:start] + src[end:] + # Tokenify: + tokens = [] + tokens_iter = tokenize.generate_tokens(io.StringIO(src).readline) + try: + for _, token_value, _, _, _ in tokens_iter: + if token_value: + tokens.append(token_value) + except tokenize.TokenError: + pass + return src, tokens diff --git a/runmanager/widgets.py b/runmanager/widgets.py new file mode 100644 index 0000000..1c39ae1 --- /dev/null +++ b/runmanager/widgets.py @@ -0,0 +1,630 @@ +##################################################################### +# # +# __main__.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the program runmanager, in the labscript # +# suite (see http://labscriptsuite.org), and is licensed under the # +# Simplified BSD License. See the license.txt file in the root of # +# the project for the full license. # +# # +##################################################################### +"""Widgets for the Runmanager GUI +""" + +import logging + +from qtutils.qt import QtCore, QtGui, QtWidgets, QT_ENV +from qtutils.qt.QtCore import pyqtSignal as Signal + +class RunmanagerColors(object): + """Singleton class that globally defines various colors for the globals view + + Colors are saved to class attributes as hex strings. + Available colors are: + + :ivar COLOR_HIGHLIGHT: Item selection highlight color, fixed to semitransparent blue + :ivar COLOR_ERROR: Item has runtime error color + :ivar COLOR_OK: Item is normal + :ivar COLOR_BOOL_ON: Item is bool = True + :ivar COLOR_BOOL_OFF: Item is bool = False + """ + _instance = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + + return cls._instance + + def __init__(self): + + if not hasattr(self, '_initialized'): + self.logger = logging.getLogger('runmanager') + + # init colors + self.update_colors_from_scheme() + + # common color definitions + self.COLOR_HIGHLIGHT = "#40308CC6" # semitransparent blue + + self._initialized = True + + def check_if_light(self): + """Helper method that return current light/dark theme state + + If using PyQt5, always returns True. + Otherwise, state is taken from `QGuiApplication` which follows OS. + """ + + # pyqt5 defaults to light and doesn't have styleHints.colorScheme() + if QT_ENV.lower() == 'pyqt5': + return True + + style_hints = QtGui.QGuiApplication.styleHints() + if style_hints.colorScheme() == QtCore.Qt.ColorScheme.Dark: + return False + else: + return True + + def update_colors_from_scheme(self): + """Method updates colors depending on current color scheme""" + + if self.check_if_light(): + self.logger.info('Setting custom light theme colors') + # use light mode colors for GroupTabs + self.COLOR_ERROR = '#F79494' # light red + self.COLOR_OK = '#A5F7C6' # light green + self.COLOR_BOOL_ON = '#63F731' # bright green + self.COLOR_BOOL_OFF = '#608060' # dark green + else: + self.logger.info('Setting custom dark theme colors') + # use dark mode colors for GroupTabs + self.COLOR_ERROR = "#BC0000" # red + self.COLOR_OK = "#2F4C00" # green + self.COLOR_BOOL_ON = "#29A300" # bright green + self.COLOR_BOOL_OFF = "#003900" # dark green + +def composite_colors(r0, g0, b0, a0, r1, g1, b1, a1): + """composite a second colour over a first with given alpha values and return the + result""" + a0 /= 255 + a1 /= 255 + a = a0 + a1 - a0 * a1 + r = (a1 * r1 + (1 - a1) * a0 * r0) / a + g = (a1 * g1 + (1 - a1) * a0 * g0) / a + b = (a1 * b1 + (1 - a1) * a0 * b0) / a + return [int(round(x)) for x in (r, g, b, 255 * a)] + +class FingerTabBarWidget(QtWidgets.QTabBar): + + """A TabBar with the tabs on the left and the text horizontal. Credit to + @LegoStormtroopr, https://gist.github.com/LegoStormtroopr/5075267. We will + promote the TabBar from the ui file to one of these.""" + + def __init__(self, parent=None, minwidth=180, minheight=30, **kwargs): + QtWidgets.QTabBar.__init__(self, parent, **kwargs) + self.minwidth = minwidth + self.minheight = minheight + self.iconPosition = kwargs.pop('iconPosition', QtWidgets.QTabWidget.West) + self._movable = None + self.tab_movable = {} + self.paint_clip = None + + def setMovable(self, movable, index=None): + """Set tabs movable on an individual basis, or set for all tabs if no + index specified""" + if index is None: + self._movable = movable + self.tab_movable = {} + QtWidgets.QTabBar.setMovable(self, movable) + else: + self.tab_movable[int(index)] = bool(movable) + + def isMovable(self, index=None): + if index is None: + if self._movable is None: + self._movable = QtWidgets.QTabBar.isMovable(self) + return self._movable + return self.tab_movable.get(index, self._movable) + + def indexAtPos(self, point): + for index in range(self.count()): + if self.tabRect(index).contains(point): + return index + + def mouseEventIndex(self, event): + if QT_ENV == 'PyQt5': + return self.indexAtPos(event.pos()) + else: + # Qt6 position returns QPointF instead of QPoint + return self.indexAtPos(event.position().toPoint()) + + def mousePressEvent(self, event): + index = self.mouseEventIndex(event) + if not self.tab_movable.get(index, self.isMovable()): + QtWidgets.QTabBar.setMovable(self, False) # disable dragging until they release the mouse + return QtWidgets.QTabBar.mousePressEvent(self, event) + + def mouseReleaseEvent(self, event): + if self.isMovable(): + # Restore this in case it was temporarily disabled by mousePressEvent + QtWidgets.QTabBar.setMovable(self, True) + return QtWidgets.QTabBar.mouseReleaseEvent(self, event) + + def tabLayoutChange(self): + total_height = 0 + for index in range(self.count()): + tabRect = self.tabRect(index) + total_height += tabRect.height() + if total_height > self.parent().height(): + # Don't paint over the top of the scroll buttons: + scroll_buttons_area_height = 2*max(self.style().pixelMetric(QtWidgets.QStyle.PM_TabBarScrollButtonWidth), + self.style().pixelMetric(QtWidgets.QStyle.PM_LayoutHorizontalSpacing)) + self.paint_clip = self.width(), self.parent().height() - scroll_buttons_area_height + else: + self.paint_clip = None + + def paintEvent(self, event): + painter = QtWidgets.QStylePainter(self) + if self.paint_clip is not None: + painter.setClipRect(0, 0, *self.paint_clip) + + option = QtWidgets.QStyleOptionTab() + for index in range(self.count()): + tabRect = self.tabRect(index) + self.initStyleOption(option, index) + painter.drawControl(QtWidgets.QStyle.CE_TabBarTabShape, option) + if not self.tabIcon(index).isNull(): + icon = self.tabIcon(index).pixmap(self.iconSize()) + alignment = QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter + tabRect.moveLeft(10) + painter.drawItemPixmap(tabRect, alignment, icon) + tabRect.moveLeft(self.iconSize().width() + 15) + else: + tabRect.moveLeft(10) + painter.drawText(tabRect, QtCore.Qt.AlignVCenter, self.tabText(index)) + if self.paint_clip is not None: + x_clip, y_clip = self.paint_clip + painter.setClipping(False) + palette = self.palette() + mid_color = palette.color(QtGui.QPalette.Mid) + painter.setPen(mid_color) + painter.drawLine(0, y_clip, x_clip, y_clip) + painter.end() + + + def tabSizeHint(self, index): + fontmetrics = QtGui.QFontMetrics(self.font()) + text_size = fontmetrics.size(QtCore.Qt.TextSingleLine, self.tabText(index)) + text_width = text_size.width() + text_height = text_size.height() + height = text_height + 15 + height = max(self.minheight, height) + width = text_width + 15 + + button = self.tabButton(index, QtWidgets.QTabBar.RightSide) + if button is not None: + height = max(height, button.height() + 7) + # Same amount of space around the button horizontally as it has vertically: + width += button.width() + height - button.height() + width = max(self.minwidth, width) + return QtCore.QSize(width, height) + + def setTabButton(self, index, geometry, button): + if not isinstance(button, TabToolButton): + raise TypeError('Not a TabToolButton, won\'t paint correctly. Use a TabToolButton') + result = QtWidgets.QTabBar.setTabButton(self, index, geometry, button) + button.move(*button.get_correct_position()) + return result + + +class TabToolButton(QtWidgets.QToolButton): + def __init__(self, *args, **kwargs): + QtWidgets.QToolButton.__init__(self, *args, **kwargs) + self.setFocusPolicy(QtCore.Qt.NoFocus) + + def paintEvent(self, event): + painter = QtWidgets.QStylePainter(self) + paint_clip = self.parent().paint_clip + if paint_clip is not None: + point = QtCore.QPoint(*paint_clip) + global_point = self.parent().mapToGlobal(point) + local_point = self.mapFromGlobal(global_point) + painter.setClipRect(0, 0, local_point.x(), local_point.y()) + option = QtWidgets.QStyleOptionToolButton() + self.initStyleOption(option) + painter.drawComplexControl(QtWidgets.QStyle.CC_ToolButton, option) + + def get_correct_position(self): + parent = self.parent() + for index in range(parent.count()): + if parent.tabButton(index, QtWidgets.QTabBar.RightSide) is self: + break + else: + raise LookupError('Tab not found') + tabRect = parent.tabRect(index) + tab_x, tab_y, tab_width, tab_height = tabRect.x(), tabRect.y(), tabRect.width(), tabRect.height() + size = self.sizeHint() + width = size.width() + height = size.height() + padding = int((tab_height - height) / 2) + correct_x = tab_x + tab_width - width - padding + correct_y = tab_y + padding + return correct_x, correct_y + + def moveEvent(self, event): + try: + correct_x, correct_y = self.get_correct_position() + except LookupError: + return # Things aren't initialised yet + if self.x() != correct_x or self.y() != correct_y: + # Move back! I shall not be moved! + self.move(correct_x, correct_y) + return QtWidgets.QToolButton.moveEvent(self, event) + + +class FingerTabWidget(QtWidgets.QTabWidget): + + """A QTabWidget equivalent which uses our FingerTabBarWidget""" + + def __init__(self, parent, *args): + QtWidgets.QTabWidget.__init__(self, parent, *args) + self.setTabBar(FingerTabBarWidget(self)) + + def addTab(self, *args, **kwargs): + closeable = kwargs.pop('closable', False) + index = QtWidgets.QTabWidget.addTab(self, *args, **kwargs) + self.setTabClosable(index, closeable) + return index + + def setTabClosable(self, index, closable): + right_button = self.tabBar().tabButton(index, QtWidgets.QTabBar.RightSide) + if closable: + if not right_button: + # Make one: + close_button = TabToolButton(self.parent()) + close_button.setIcon(QtGui.QIcon(':/qtutils/fugue/cross')) + self.tabBar().setTabButton(index, QtWidgets.QTabBar.RightSide, close_button) + close_button.clicked.connect(lambda: self._on_close_button_clicked(close_button)) + else: + if right_button: + # Get rid of it: + self.tabBar().setTabButton(index, QtWidgets.QTabBar.RightSide, None) + + def _on_close_button_clicked(self, button): + for index in range(self.tabBar().count()): + if self.tabBar().tabButton(index, QtWidgets.QTabBar.RightSide) is button: + self.tabCloseRequested.emit(index) + break + + +class ItemView(object): + """Mixin for QTableView and QTreeView that emits a custom signal leftClicked(index) + after a left click on a valid index, and doubleLeftClicked(index) (in addition) on + double click. Also has modified tab and arrow key behaviour and custom selection + highlighting.""" + leftClicked = Signal(QtCore.QModelIndex) + doubleLeftClicked = Signal(QtCore.QModelIndex) + + + def __init__(self, *args): + super(ItemView, self).__init__(*args) + self._pressed_index = None + self._double_click = False + self.setAutoScroll(False) + palette = self.palette() + for group in [QtGui.QPalette.Active, QtGui.QPalette.Inactive]: + palette.setColor( + group, + QtGui.QPalette.Highlight, + QtGui.QColor(RunmanagerColors().COLOR_HIGHLIGHT) + ) + palette.setColor( + group, + QtGui.QPalette.HighlightedText, + palette.color(QtGui.QPalette.WindowText) + ) + self.setPalette(palette) + + def mouseEventIndex(self, event): + if QT_ENV == 'PyQt5': + return self.indexAt(event.pos()) + else: + # Qt6 returns QPointF instead of QPoint + return self.indexAt(event.position().toPoint()) + + def mousePressEvent(self, event): + result = super(ItemView, self).mousePressEvent(event) + index = self.mouseEventIndex(event) + if event.button() == QtCore.Qt.LeftButton and index.isValid(): + self._pressed_index = self.mouseEventIndex(event) + return result + + def leaveEvent(self, event): + result = super(ItemView, self).leaveEvent(event) + self._pressed_index = None + self._double_click = False + return result + + def mouseDoubleClickEvent(self, event): + # Ensure our left click event occurs regardless of whether it is the + # second click in a double click or not + result = super(ItemView, self).mouseDoubleClickEvent(event) + index = self.mouseEventIndex(event) + if event.button() == QtCore.Qt.LeftButton and index.isValid(): + self._pressed_index = self.mouseEventIndex(event) + self._double_click = True + return result + + def mouseReleaseEvent(self, event): + result = super(ItemView, self).mouseReleaseEvent(event) + index = self.mouseEventIndex(event) + if event.button() == QtCore.Qt.LeftButton and index.isValid() and index == self._pressed_index: + self.leftClicked.emit(index) + if self._double_click: + self.doubleLeftClicked.emit(index) + self._pressed_index = None + self._double_click = False + return result + + def keyPressEvent(self, event): + if event.key() in [QtCore.Qt.Key_Space, QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return]: + item = self.model().itemFromIndex(self.currentIndex()) + if item.isEditable(): + # Space/enter edits editable items: + self.edit(self.currentIndex()) + else: + # Space/enter on non-editable items simulates a left click: + self.leftClicked.emit(self.currentIndex()) + return super(ItemView, self).keyPressEvent(event) + + def moveCursor(self, cursor_action, keyboard_modifiers): + current_index = self.currentIndex() + current_row, current_column = current_index.row(), current_index.column() + if cursor_action == QtWidgets.QAbstractItemView.MoveUp: + return current_index.sibling(current_row - 1, current_column) + elif cursor_action == QtWidgets.QAbstractItemView.MoveDown: + return current_index.sibling(current_row + 1, current_column) + elif cursor_action == QtWidgets.QAbstractItemView.MoveLeft: + return current_index.sibling(current_row, current_column - 1) + elif cursor_action == QtWidgets.QAbstractItemView.MoveRight: + return current_index.sibling(current_row, current_column + 1) + elif cursor_action == QtWidgets.QAbstractItemView.MovePrevious: + return current_index.sibling(current_row, current_column - 1) + elif cursor_action == QtWidgets.QAbstractItemView.MoveNext: + return current_index.sibling(current_row, current_column + 1) + else: + return super(ItemView, self).moveCursor(cursor_action, keyboard_modifiers) + + +class TreeView(ItemView, QtWidgets.QTreeView): + """Treeview version of our customised ItemView""" + def __init__(self, parent=None): + super(TreeView, self).__init__(parent) + # Set columns to their minimum size, disabling resizing. Caller may still + # configure a specific section to stretch: + self.header().setSectionResizeMode( + QtWidgets.QHeaderView.ResizeToContents + ) + self.setItemDelegate(ItemDelegate(self)) + + +class TableView(ItemView, QtWidgets.QTableView): + """TableView version of our customised ItemView""" + def __init__(self, parent=None): + super(TableView, self).__init__(parent) + # Set rows and columns to the minimum size, disabling interactive resizing. + # Caller may still configure a specific column to stretch: + self.verticalHeader().setSectionResizeMode( + QtWidgets.QHeaderView.ResizeToContents + ) + self.horizontalHeader().setSectionResizeMode( + QtWidgets.QHeaderView.ResizeToContents + ) + self.horizontalHeader().sectionResized.connect(self.on_column_resized) + self.setItemDelegate(ItemDelegate(self)) + self.verticalHeader().hide() + self.setShowGrid(False) + self.horizontalHeader().setHighlightSections(False) + + def on_column_resized(self, col): + for row in range(self.model().rowCount()): + self.resizeRowToContents(row) + + +class AlternatingColorModel(QtGui.QStandardItemModel): + + def __init__(self, view): + QtGui.QStandardItemModel.__init__(self) + # How much darker in each channel is the alternate base color compared + # to the base color? + self.view = view + + # A cache, store brushes so we don't have to recalculate them. Is faster. + self.bg_brushes = {} + + def get_bgbrush(self, normal_brush, alternate, selected): + """Get cell colour as a function of its ordinary colour, whether it is on an odd + row, and whether it is selected.""" + normal_rgb = normal_brush.color().getRgb() if normal_brush is not None else None + try: + return self.bg_brushes[normal_rgb, alternate, selected] + except KeyError: + pass + # Get the colour of the cell with alternate row shading: + normal_color = self.view.palette().color( + QtGui.QPalette.ColorGroup.Disabled, + QtGui.QPalette.ColorRole.Base + ) + alternate_color = self.view.palette().color( + QtGui.QPalette.ColorGroup.Disabled, + QtGui.QPalette.ColorRole.AlternateBase + ) + if normal_rgb is None: + # No colour has been set. Use palette colours: + if alternate: + bg_color = alternate_color + else: + bg_color = normal_color + else: + bg_color = normal_brush.color() + if alternate: + # Modify alternate rows: + r, g, b, a = normal_rgb + nr, ng, nb, na = normal_color.getRgb() + ar, ag, ab, aa = alternate_color.getRgb() + alt_r = min(max(r + ar - nr, 0), 255) + alt_g = min(max(g + ag - ng, 0), 255) + alt_b = min(max(b + ab - nb, 0), 255) + alt_a = min(max(a + aa - na, 0), 255) + bg_color = QtGui.QColor(alt_r, alt_g, alt_b, alt_a) + + # If parent is a TableView, we handle selection highlighting as part of the + # background colours: + if selected and isinstance(self.view, QtWidgets.QTableView): + # Overlay highlight colour: + r_s, g_s, b_s, a_s = QtGui.QColor(RunmanagerColors().COLOR_HIGHLIGHT).getRgb() + r_0, g_0, b_0, a_0 = bg_color.getRgb() + rgb = composite_colors(r_0, g_0, b_0, a_0, r_s, g_s, b_s, a_s) + bg_color = QtGui.QColor(*rgb) + + brush = QtGui.QBrush(bg_color) + self.bg_brushes[normal_rgb, alternate, selected] = brush + return brush + + def data(self, index, role): + """When background color data is being requested, returns modified colours for + every second row, according to the palette of the view. This has the effect of + making the alternate colours visible even when custom colors have been set - the + same shading will be applied to the custom colours. Only really looks sensible + when the normal and alternate colors are similar. Also applies selection + highlight colour (using RunmanagerColors().COLOR_HIGHLIGHT), similarly with alternate-row + shading, for the case of a QTableView.""" + if role == QtCore.Qt.BackgroundRole: + normal_brush = QtGui.QStandardItemModel.data(self, index, QtCore.Qt.BackgroundRole) + selected = index in self.view.selectedIndexes() + alternate = index.row() % 2 + return self.get_bgbrush(normal_brush, alternate, selected) + return QtGui.QStandardItemModel.data(self, index, role) + + +class Editor(QtWidgets.QTextEdit): + """Popup editor with word wrapping and automatic resizing.""" + + def __init__(self, parent): + QtWidgets.QTextEdit.__init__(self, parent) + self.setWordWrapMode(QtGui.QTextOption.WordWrap) + self.setAcceptRichText(False) + self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff) + self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff) + self.textChanged.connect(self.update_size) + self.initial_height = None + palette = self.palette() + for group in [QtGui.QPalette.Active, QtGui.QPalette.Inactive]: + palette.setColor( + group, + QtGui.QPalette.Highlight, + QtGui.QColor(RunmanagerColors().COLOR_HIGHLIGHT) + ) + palette.setColor( + group, + QtGui.QPalette.HighlightedText, + palette.color(QtGui.QPalette.WindowText) + ) + self.setPalette(palette) + + def update_size(self): + if self.initial_height is not None: + # Temporarily shrink back to the initial height, just so that the document + # size below returns the preferred size rather than the current size. + # QTextDocument doesn't have a sizeHint of minimumSizeHint method, so this + # is the best we can do to get its minimum size. + self.setFixedHeight(self.initial_height) + preferred_height = self.document().size().toSize().height() + # Do not shrink smaller than the initial height: + if self.initial_height is not None and preferred_height >= self.initial_height: + self.setFixedHeight(preferred_height) + + def resizeEvent(self, event): + result = QtWidgets.QTextEdit.resizeEvent(self, event) + # Record the initial height after it is first set: + if self.initial_height is None: + self.initial_height = self.height() + return result + + + +class ItemDelegate(QtWidgets.QStyledItemDelegate): + + """An item delegate with a larger row height and column width, faint grey vertical + lines between columns, and a custom editor for handling multi-line data""" + MIN_ROW_HEIGHT = 22 + EXTRA_ROW_HEIGHT = 6 + EXTRA_COL_WIDTH = 20 + + def __init__(self, *args, **kwargs): + QtWidgets.QStyledItemDelegate.__init__(self, *args, **kwargs) + self._pen = QtGui.QPen() + self._pen.setWidth(1) + self._pen.setColor(QtGui.QColor.fromRgb(128, 128, 128, 64)) + + def sizeHint(self, *args): + size = QtWidgets.QStyledItemDelegate.sizeHint(self, *args) + if size.height() <= self.MIN_ROW_HEIGHT: + height = self.MIN_ROW_HEIGHT + else: + # Esnure cells with multiple lines of text still have some padding: + height = size.height() + self.EXTRA_ROW_HEIGHT + return QtCore.QSize(size.width() + self.EXTRA_COL_WIDTH, height) + + def paint(self, painter, option, index): + if isinstance(self.parent(), QtWidgets.QTableView): + # Disable rendering of selection highlight for TableViews, they handle + # it themselves with the background colour data: + option.state &= ~(QtWidgets.QStyle.State_Selected) + QtWidgets.QStyledItemDelegate.paint(self, painter, option, index) + if index.column() > 0: + painter.setPen(self._pen) + painter.drawLine(option.rect.topLeft(), option.rect.bottomLeft()) + + def eventFilter(self, obj, event): + """Filter events before they get to the editor, so that editing is ended when + the user presses tab, shift-tab or enter (which otherwise would not end editing + in a QTextEdit).""" + if event.type() == QtCore.QEvent.KeyPress: + if event.key() in [QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return]: + # Allow shift-enter + if not event.modifiers() & QtCore.Qt.ShiftModifier: + self.commitData.emit(obj) + self.closeEditor.emit(obj) + return True + elif event.key() == QtCore.Qt.Key_Tab: + self.commitData.emit(obj) + self.closeEditor.emit(obj, QtWidgets.QStyledItemDelegate.EditNextItem) + return True + elif event.key() == QtCore.Qt.Key_Backtab: + self.commitData.emit(obj) + self.closeEditor.emit(obj, QtWidgets.QStyledItemDelegate.EditPreviousItem) + return True + return QtWidgets.QStyledItemDelegate.eventFilter(self, obj, event) + + def createEditor(self, parent, option, index): + return Editor(parent) + + def setEditorData(self, editor, index): + editor.setPlainText(index.data()) + font = index.data(QtCore.Qt.FontRole) + default_font = QtWidgets.QApplication.instance().font(self.parent()) + if font is None: + font = default_font + font.setPointSize(default_font.pointSize()) + editor.setFont(font) + font_height = QtGui.QFontMetrics(font).height() + padding = (self.MIN_ROW_HEIGHT - font_height) / 2 - 1 + editor.document().setDocumentMargin(padding) + editor.selectAll() + + def setModelData(self, editor, model, index): + model.setData(index, editor.toPlainText())